Other/data-migration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

代码调整

Browse Source
This commit is contained in:
hongawen
2024-01-17 09:54:21 +08:00
parent 661f8d2e06
commit 47d31095b9
10 changed files with 96 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
oracle-data/oracle-source/src/main/java/com/njcn/oracle/bo/po/JobDetail.java
View File

@@ -6,8 +6,11 @@ import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.github.jeffreyning.mybatisplus.anno.MppMultiId;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.xml.ws.soap.Addressing;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -23,6 +26,8 @@ import java.util.Objects;
*/
@TableName(value = "JOB_DETAIL")
@Data
@NoArgsConstructor
@AllArgsConstructor
public class JobDetail {
/**
* 指标表名
@@ -75,5 +80,17 @@ public class JobDetail {
return Objects.equals(tableName, jobDetail.tableName) && Objects.equals(excuteDate, jobDetail.excuteDate);
}
public JobDetail(String tableName, LocalDate excuteDate, Integer state, Integer rowCount, LocalDateTime updateTime) {
this.tableName = tableName;
this.excuteDate = excuteDate;
this.state = state;
this.rowCount = rowCount;
this.updateTime = updateTime;
}
public JobDetail(String tableName, LocalDate excuteDate, Integer rowCount) {
this.tableName = tableName;
this.excuteDate = excuteDate;
this.rowCount = rowCount;
}
}

3
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/JobDetailService.java
View File

@@ -6,6 +6,7 @@ import com.njcn.oracle.bo.param.JobQueryParam;
import com.njcn.oracle.bo.po.JobDetail;
import com.baomidou.mybatisplus.extension.service.IService;
import java.time.LocalDate;
import java.util.List;
/**
@@ -17,7 +18,7 @@ import java.util.List;
*/
public interface JobDetailService extends IMppService<JobDetail> {
JobDetail select(JobDetail jobDetail);
JobDetail select(String tableName, LocalDate date);
IPage<JobDetail> selectByParam(JobQueryParam jobQueryParam);

6
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/impl/JobDetailServiceImpl.java
View File

@@ -36,10 +36,10 @@ public class JobDetailServiceImpl extends MppServiceImpl<JobDetailMapper, JobDet
@Override
public JobDetail select(JobDetail jobDetail) {
public JobDetail select(String tableName, LocalDate date) {
return this.lambdaQuery()
.eq(JobDetail::getTableName, jobDetail.getTableName())
.eq(JobDetail::getExcuteDate, jobDetail.getExcuteDate())
.eq(JobDetail::getTableName, tableName)
.eq(JobDetail::getExcuteDate, date)
.one();
}

62
oracle-data/oracle-target/src/main/java/com/njcn/oracle/service/impl/OracleServiceImpl.java
View File

@@ -61,7 +61,13 @@ public class OracleServiceImpl implements IOracleService {
throw new RuntimeException(e);
}
List<LocalDate> dateList = LocalDateUtil.getDateList(dataAsynParam.getStartTime(), dataAsynParam.getEndTime());
dateList.forEach(date -> {
for (LocalDate date : dateList) {
//日志记录
JobDetail jobDetail = jobDetailService.select(tableName, date);
if (Objects.nonNull(jobDetail) && (jobDetail.getState() == 1 || jobDetail.getState() == 0)) {
//如果该指标当前天已执行或正在执行，直接跳出循环
break;
}
//程序监听
StopWatch stopWatch = new StopWatch();
stopWatch.start();
@@ -84,39 +90,6 @@ public class OracleServiceImpl implements IOracleService {
System.out.println(tableName + "查到" + size + "条数据后总堆内存为：" + runtime.totalMemory() / (1024 * 1024) + " MB");
System.out.println(tableName + "查到" + size + "条数据后已用堆内存为：" + (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024) + " MB");
System.out.println(tableName + "查到" + size + "条数据后空闲堆内存为：" + runtime.freeMemory() / (1024 * 1024) + " MB");
//日志记录
JobDetail jobDetail = new JobDetail();
jobDetail.setTableName(tableName);
jobDetail.setExcuteDate(date);
jobDetail = jobDetailService.select(jobDetail);
if (Objects.nonNull(jobDetail)) {
if (jobDetail.getState() == 2 && i == 0 && size != 0) {
//第一片执行时返现是失败的，则再次执行
jobDetail.setState(0);
jobDetail.setRowCount(size);
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.updateByMultiId(jobDetail);
} else if (jobDetail.getState() == 0 && i > 0 && jobDetail.getRowCount() + size != 0) {
// 处理中，后续时间片的处理，累加记录数
jobDetail.setState(0);
jobDetail.setRowCount(jobDetail.getRowCount() + size);
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.updateByMultiId(jobDetail);
} else {
System.gc();
break;
}
} else {
if (size > 0) {
jobDetail = new JobDetail();
jobDetail.setTableName(tableName);
jobDetail.setExcuteDate(date);
jobDetail.setState(0);
jobDetail.setRowCount(size);
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.save(jobDetail);
}
}
try {
if (CollectionUtil.isNotEmpty(weakReferenceData.get())) {
//执行目标库的数据处理
@@ -124,6 +97,20 @@ public class OracleServiceImpl implements IOracleService {
executor.insertBatchByDB(weakReferenceData.get());
//最后一片时修改状态
}
//手动执行GC
System.gc();
if (Objects.nonNull(jobDetail)) {
if (i == 0) {
jobDetail.setState(0);
jobDetail.setRowCount(0);
}
jobDetail.setRowCount(jobDetail.getRowCount() + size);
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.updateByMultiId(jobDetail);
} else {
jobDetail = new JobDetail(tableName, date, 0, size, LocalDateTime.now());
jobDetailService.save(jobDetail);
}
if (i + 1 == slice && Objects.nonNull(jobDetail)) {
stopWatch.stop();
jobDetail.setState(1);
@@ -133,6 +120,9 @@ public class OracleServiceImpl implements IOracleService {
}
} catch (Exception exception) {
exception.printStackTrace();
if (Objects.isNull(jobDetail)) {
jobDetail = new JobDetail(tableName, date, size);
}
jobDetail.setState(2);
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.updateByMultiId(jobDetail);
@@ -142,7 +132,9 @@ public class OracleServiceImpl implements IOracleService {
System.out.println("执行后已用堆内存为：" + (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024) + " MB");
System.out.println("执行后空闲堆内存为：" + runtime.freeMemory() / (1024 * 1024) + " MB");
}
});
}
});
}
}

2
oracle-data/oracle-target/src/main/resources/application.yml
View File

@@ -2,7 +2,7 @@ server:
port: 8091
spring:
application:
name: oracle-data
name: oracle-oracle
autoconfigure:
exclude: com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure
datasource: