Other/data-migration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

代码优化调整

Browse Source
This commit is contained in:
hongawen
2024-01-15 20:27:55 +08:00
parent 7b31e0376c
commit c3f7491598
13 changed files with 208 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
oracle-data/oracle-source/src/main/java/com/njcn/oracle/bo/po/JobDetail.java
View File

@@ -53,11 +53,13 @@ public class JobDetail {
*/
@TableField(value = "\"ROW_COUNT\"")
private Integer rowCount;
@JsonFormat(
pattern = "yyyy-MM-dd HH:mm:ss"
)
@TableField(value = "UPDATE_TIME")
private LocalDateTime updateTime;
@TableField(value = "DURATION")
private Double duration;

1
oracle-data/oracle-source/src/main/java/com/njcn/oracle/mybatis/service/impl/ReplenishMybatisServiceImpl.java
View File

@@ -22,7 +22,6 @@ public abstract class ReplenishMybatisServiceImpl<M extends BatchBaseMapper<T>,
@Override
@Transactional(rollbackFor = Exception.class)
public void insertBatchBySlice(List<T> data, int size) throws RuntimeException{
int totalCount = data.size();
int idxLimit = Math.min(size, totalCount);
List<T> dataTemp = new ArrayList<>(data);

19
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/DataSyncService.java
View File

@@ -1,19 +0,0 @@
package com.njcn.oracle.service;
import com.njcn.oracle.bo.param.DataAsynParam;
import com.njcn.oracle.mybatis.service.IReplenishMybatisService;
import java.time.LocalDate;
/**
* Description:
* Date: 2024/1/8 16:16【需求编号】
*
* @author clam
* @version V1.0.0
*/
public interface DataSyncService {
void dataSync(IReplenishMybatisService executor, String temp, LocalDate migration);
void dataBacthSysc(DataAsynParam dataAsynParam);
}

5
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/JobDetailService.java
View File

@@ -9,7 +9,6 @@ import com.baomidou.mybatisplus.extension.service.IService;
import java.util.List;
/**
*
* Description:
* Date: 2024/1/8 12:27【需求编号】
*
@@ -18,9 +17,9 @@ import java.util.List;
*/
public interface JobDetailService extends IMppService<JobDetail> {
JobDetail select(JobDetail jobDetail);
JobDetail select(JobDetail jobDetail);
IPage<JobDetail> selectByParam(JobQueryParam jobQueryParam);
IPage<JobDetail> selectByParam(JobQueryParam jobQueryParam);
boolean jobRemove(JobDetail jobDetail);
}

112
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/impl/DataSyncServiceImpl.java
View File

@@ -1,112 +0,0 @@
package com.njcn.oracle.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.LocalDateTimeUtil;
import cn.hutool.extra.spring.SpringUtil;
import com.njcn.oracle.bo.param.DataAsynParam;
import com.njcn.oracle.bo.param.MigrationParam;
import com.njcn.oracle.bo.po.JobDetail;
import com.njcn.oracle.mybatis.service.IReplenishMybatisService;
import com.njcn.oracle.service.DataSyncService;
import com.njcn.oracle.service.JobDetailService;
import com.njcn.oracle.util.LocalDateUtil;
import lombok.RequiredArgsConstructor;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Objects;
/**
* Description:
* Date: 2024/1/8 16:16【需求编号】
*
* @author clam
* @version V1.0.0
*/
@Service
@RequiredArgsConstructor
public class DataSyncServiceImpl implements DataSyncService {
private final JobDetailService jobDetailService;
@Override
public void dataSync(IReplenishMybatisService executor, String temp, LocalDate date) {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
LocalDateTime localDateTime = date.atStartOfDay();
LocalDateTime tempStartTime = LocalDateTimeUtil.beginOfDay(localDateTime);
LocalDateTime tempEndTime = LocalDateTimeUtil.endOfDay(localDateTime);
MigrationParam migration = new MigrationParam();
migration.setStartTime(tempStartTime);
migration.setEndTime(tempEndTime);
List list = executor.queryData(migration);
JobDetail jobDetail = new JobDetail();
jobDetail.setTableName(temp);
jobDetail.setExcuteDate(date);
JobDetail select = jobDetailService.select(jobDetail);
if (Objects.nonNull(select)){
if (select.getState()==2) {
jobDetail.setState(0);
jobDetail.setRowCount(list.size());
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.updateByMultiId(jobDetail);
}else {
return;
}
}else {
jobDetail.setState(0);
jobDetail.setRowCount(list.size());
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.save(jobDetail);
}
try {
if(CollectionUtil.isNotEmpty(list)){
executor.clearTargetData(migration);
executor.insertBatchByDB(list);
}
stopWatch.stop();
jobDetail.setState(1);
jobDetail.setDuration(stopWatch.getTotalTimeSeconds());
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.updateByMultiId(jobDetail);
}catch (
Exception e
){
jobDetail.setState(2);
jobDetail.setUpdateTime(LocalDateTime.now());
jobDetailService.updateByMultiId(jobDetail);
}
}
@Override
@Async
public void dataBacthSysc(DataAsynParam dataAsynParam) {
List<String> tableNames = dataAsynParam.getTableNames();
tableNames.stream().forEach(temp->{
IReplenishMybatisService executor = null;
try {
executor = (IReplenishMybatisService) SpringUtil.getBean(Class.forName("com.njcn.oracle.service.impl." + temp + "ServiceImpl"));
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
List<LocalDate> dateList = LocalDateUtil.getDateList(dataAsynParam.getStartTime(), dataAsynParam.getEndTime());
IReplenishMybatisService finalExecutor = executor;
dateList.forEach(date->{
this.dataSync(finalExecutor,temp,date);
});
});
}
}

37
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/impl/JobDetailServiceImpl.java
View File

@@ -10,6 +10,7 @@ import com.njcn.oracle.bo.param.ServiceTypeEnum;
import com.njcn.oracle.util.LocalDateUtil;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.time.LocalDate;
import java.util.ArrayList;
@@ -24,7 +25,6 @@ import com.njcn.oracle.service.JobDetailService;
import org.springframework.util.CollectionUtils;
/**
*
* Description:
* Date: 2024/1/8 12:27【需求编号】
*
@@ -33,14 +33,15 @@ import org.springframework.util.CollectionUtils;
*/
@Service
@DS(("target"))
public class JobDetailServiceImpl extends MppServiceImpl<JobDetailMapper, JobDetail> implements JobDetailService{
public class JobDetailServiceImpl extends MppServiceImpl<JobDetailMapper, JobDetail> implements JobDetailService {
@Override
public JobDetail select(JobDetail jobDetail) {
JobDetail jobDetail1 = this.lambdaQuery().eq(JobDetail::getTableName,jobDetail.getTableName()).
eq(JobDetail::getExcuteDate,jobDetail.getExcuteDate()).one();
return jobDetail1;
return this.lambdaQuery()
.eq(JobDetail::getTableName, jobDetail.getTableName())
.eq(JobDetail::getExcuteDate, jobDetail.getExcuteDate())
.one();
}
@Override
@@ -52,21 +53,21 @@ public class JobDetailServiceImpl extends MppServiceImpl<JobDetailMapper, JobDet
in(!CollectionUtils.isEmpty(jobQueryParam.getTableNames()), JobDetail::getTableName, jobQueryParam.getTableNames());
List<JobDetail> jobDetails = this.getBaseMapper().selectList(queryWrapper);
List<String> tableNames = new ArrayList<>();
if(CollectionUtils.isEmpty(jobQueryParam.getTableNames())){
List<String> tableNames;
if (CollectionUtils.isEmpty(jobQueryParam.getTableNames())) {
tableNames = ServiceTypeEnum.getExecutableTypes();
}else {
} else {
tableNames = jobQueryParam.getTableNames();
}
List<LocalDate> dateList = LocalDateUtil.getDateList(jobQueryParam.getStartTime(), jobQueryParam.getEndTime());
List<String> finalTableNames = tableNames;
dateList.stream().forEach(localDate -> {
finalTableNames.stream().forEach(temp->{
finalTableNames.stream().forEach(temp -> {
JobDetail jobDetail = new JobDetail();
jobDetail.setTableName(temp);
jobDetail.setExcuteDate(localDate);
if (jobDetails.contains(jobDetail)){
if (jobDetails.contains(jobDetail)) {
return;
}
jobDetail.setTableName_CN(ServiceTypeEnum.getValueByCode(temp));
@@ -75,23 +76,23 @@ public class JobDetailServiceImpl extends MppServiceImpl<JobDetailMapper, JobDet
});
});
List<JobDetail> jobDetailIPage= jobDetails.stream().sorted(Comparator.comparing(JobDetail::getExcuteDate).reversed()).
List<JobDetail> jobDetailIPage = jobDetails.stream().sorted(Comparator.comparing(JobDetail::getExcuteDate).reversed()).
skip((jobQueryParam.getCurrentPage() - 1) * jobQueryParam.getPageSize()).
limit(jobQueryParam.getPageSize()).collect(Collectors.toList());
jobDetailIPage.stream().forEach(temp->temp.setTableName_CN(ServiceTypeEnum.getValueByCode(temp.getTableName())));
jobDetailIPage.stream().forEach(temp -> temp.setTableName_CN(ServiceTypeEnum.getValueByCode(temp.getTableName())));
page.setRecords(jobDetailIPage);
page.setTotal(Long.parseLong(jobDetails.size()+""));
page.setTotal(Long.parseLong(jobDetails.size() + ""));
return page;
}
@Override
public boolean jobRemove(JobDetail jobDetail) {
QueryWrapper<JobDetail> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(JobDetail::getTableName,jobDetail.getTableName()).
eq(JobDetail::getExcuteDate,jobDetail.getExcuteDate());
boolean remove = this.remove(queryWrapper);
return remove;
queryWrapper.lambda()
.eq(JobDetail::getTableName, jobDetail.getTableName())
.eq(JobDetail::getExcuteDate, jobDetail.getExcuteDate());
return this.remove(queryWrapper);
}

2
oracle-data/oracle-source/src/main/java/com/njcn/oracle/util/LocalDateUtil.java
View File

@@ -16,11 +16,9 @@ public class LocalDateUtil {
public static List<LocalDate> getDateList(LocalDate startDate, LocalDate endDate) {
List<LocalDate> dateList = new ArrayList<>();
long daysBetween = ChronoUnit.DAYS.between(startDate, endDate);
for (long i = 0; i <= daysBetween; i++) {
dateList.add(startDate.plusDays(i));
}
return dateList;
}
}