Other/data-migration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

代码调整

Browse Source
This commit is contained in:
hongawen
2024-01-17 09:54:21 +08:00
parent 661f8d2e06
commit 47d31095b9
10 changed files with 96 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
influx-data/influx-source/src/main/java/com/njcn/influx/bo/po/JobDetailInfluxDB.java
View File

@@ -4,7 +4,9 @@ import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.github.jeffreyning.mybatisplus.anno.MppMultiId;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -20,6 +22,8 @@ import java.util.Objects;
*/
@TableName(value = "JOB_DETAIL_INFLUXDB")
@Data
@NoArgsConstructor
@AllArgsConstructor
public class JobDetailInfluxDB {
/**
* 指标表名
@@ -73,4 +77,17 @@ public class JobDetailInfluxDB {
}
public JobDetailInfluxDB(String tableName, LocalDate excuteDate, Integer state, Integer rowCount, LocalDateTime updateTime) {
this.tableName = tableName;
this.excuteDate = excuteDate;
this.state = state;
this.rowCount = rowCount;
this.updateTime = updateTime;
}
public JobDetailInfluxDB(String tableName, LocalDate excuteDate, Integer rowCount) {
this.tableName = tableName;
this.excuteDate = excuteDate;
this.rowCount = rowCount;
}
}

5
influx-data/influx-source/src/main/java/com/njcn/influx/service/JobDetailInfluxDBService.java
View File

@@ -5,6 +5,8 @@ import com.github.jeffreyning.mybatisplus.service.IMppService;
import com.njcn.influx.bo.po.JobDetailInfluxDB;
import com.njcn.oracle.bo.param.JobQueryParam;
import java.time.LocalDate;
/**
* Description:
* Date: 2024/1/8 12:27【需求编号】
@@ -14,9 +16,10 @@ import com.njcn.oracle.bo.param.JobQueryParam;
*/
public interface JobDetailInfluxDBService extends IMppService<JobDetailInfluxDB> {
JobDetailInfluxDB select(JobDetailInfluxDB jobDetail);
JobDetailInfluxDB select(String tableName, LocalDate date);
IPage<JobDetailInfluxDB> selectByParam(JobQueryParam jobQueryParam);
boolean jobRemove(JobDetailInfluxDB jobDetail);
}

6
influx-data/influx-source/src/main/java/com/njcn/influx/service/impl/JobDetailInfluxDBServiceImpl.java
View File

@@ -30,10 +30,10 @@ public class JobDetailInfluxDBServiceImpl extends MppServiceImpl<JobDetailInflux
@Override
public JobDetailInfluxDB select(JobDetailInfluxDB jobDetail) {
public JobDetailInfluxDB select(String tableName, LocalDate date) {
return this.lambdaQuery()
.eq(JobDetailInfluxDB::getTableName, jobDetail.getTableName())
.eq(JobDetailInfluxDB::getExcuteDate, jobDetail.getExcuteDate())
.eq(JobDetailInfluxDB::getTableName, tableName)
.eq(JobDetailInfluxDB::getExcuteDate, date)
.one();
}

52
influx-data/influx-source/src/main/java/com/njcn/influx/service/impl/OracleToInfluxDBServiceImpl.java
View File

@@ -70,7 +70,13 @@ public class OracleToInfluxDBServiceImpl implements OracleToInfluxDBService {
throw new RuntimeException(e);
}
List<LocalDate> dateList = LocalDateUtil.getDateList(dataAsynParam.getStartTime(), dataAsynParam.getEndTime());
dateList.forEach(date -> {
for (LocalDate date : dateList) {
//日志记录
JobDetailInfluxDB jobDetailInfluxDB = jobDetailInfluxDBService.select(tableName, date);
if (Objects.nonNull(jobDetailInfluxDB) && (jobDetailInfluxDB.getState() == 1 || jobDetailInfluxDB.getState() == 0)) {
//如果该指标当前天已执行或正在执行，直接跳出循环
break;
}
//程序监听
StopWatch stopWatch = new StopWatch();
stopWatch.start();
@@ -93,11 +99,6 @@ public class OracleToInfluxDBServiceImpl implements OracleToInfluxDBService {
System.out.println(tableName + "查到" + size + "条数据后总堆内存为：" + runtime.totalMemory() / (1024 * 1024) + " MB");
System.out.println(tableName + "查到" + size + "条数据后已用堆内存为：" + (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024) + " MB");
System.out.println(tableName + "查到" + size + "条数据后空闲堆内存为：" + runtime.freeMemory() / (1024 * 1024) + " MB");
//日志记录
JobDetailInfluxDB jobDetailInfluxDB = new JobDetailInfluxDB();
jobDetailInfluxDB.setTableName(tableName);
jobDetailInfluxDB.setExcuteDate(date);
jobDetailInfluxDB = jobDetailInfluxDBService.select(jobDetailInfluxDB);
try {
if (CollectionUtil.isNotEmpty(weakReferenceData.get())) {
//执行目标库的数据处理
@@ -110,7 +111,7 @@ public class OracleToInfluxDBServiceImpl implements OracleToInfluxDBService {
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
Method method = null;
Method method;
try {
method = clazz.getDeclaredMethod("oralceToInfluxDB", clazz2);
} catch (NoSuchMethodException e) {
@@ -133,33 +134,19 @@ public class OracleToInfluxDBServiceImpl implements OracleToInfluxDBService {
size = list1.size();
//最后一片时修改状态
}
//手动执行GC
System.gc();
if (Objects.nonNull(jobDetailInfluxDB)) {
if (jobDetailInfluxDB.getState() == 2 && i == 0 && size != 0) {
//第一片执行时返现是失败的，则再次执行
if (i == 0) {
jobDetailInfluxDB.setState(0);
jobDetailInfluxDB.setRowCount(size);
jobDetailInfluxDB.setUpdateTime(LocalDateTime.now());
jobDetailInfluxDBService.updateByMultiId(jobDetailInfluxDB);
} else if (jobDetailInfluxDB.getState() == 0 && i > 0 && jobDetailInfluxDB.getRowCount() + size != 0) {
// 处理中，后续时间片的处理，累加记录数
jobDetailInfluxDB.setState(0);
jobDetailInfluxDB.setRowCount(jobDetailInfluxDB.getRowCount() + size);
jobDetailInfluxDB.setUpdateTime(LocalDateTime.now());
jobDetailInfluxDBService.updateByMultiId(jobDetailInfluxDB);
} else {
System.gc();
break;
jobDetailInfluxDB.setRowCount(0);
}
jobDetailInfluxDB.setRowCount(jobDetailInfluxDB.getRowCount() + size);
jobDetailInfluxDB.setUpdateTime(LocalDateTime.now());
jobDetailInfluxDBService.updateByMultiId(jobDetailInfluxDB);
} else {
if (size > 0) {
jobDetailInfluxDB = new JobDetailInfluxDB();
jobDetailInfluxDB.setTableName(tableName);
jobDetailInfluxDB.setExcuteDate(date);
jobDetailInfluxDB.setState(0);
jobDetailInfluxDB.setRowCount(size);
jobDetailInfluxDB.setUpdateTime(LocalDateTime.now());
jobDetailInfluxDBService.save(jobDetailInfluxDB);
}
jobDetailInfluxDB = new JobDetailInfluxDB(tableName, date, 0, size, LocalDateTime.now());
jobDetailInfluxDBService.save(jobDetailInfluxDB);
}
if (i + 1 == slice && Objects.nonNull(jobDetailInfluxDB)) {
stopWatch.stop();
@@ -170,6 +157,9 @@ public class OracleToInfluxDBServiceImpl implements OracleToInfluxDBService {
}
} catch (Exception exception) {
exception.printStackTrace();
if (Objects.isNull(jobDetailInfluxDB)) {
jobDetailInfluxDB = new JobDetailInfluxDB(tableName, date, size);
}
jobDetailInfluxDB.setState(2);
jobDetailInfluxDB.setUpdateTime(LocalDateTime.now());
jobDetailInfluxDBService.updateByMultiId(jobDetailInfluxDB);
@@ -179,7 +169,7 @@ public class OracleToInfluxDBServiceImpl implements OracleToInfluxDBService {
System.out.println("执行后已用堆内存为：" + (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024) + " MB");
System.out.println("执行后空闲堆内存为：" + runtime.freeMemory() / (1024 * 1024) + " MB");
}
});
}
});
}
}

2
influx-data/influx-target/src/main/resources/application.yml
View File

@@ -13,7 +13,7 @@ spring:
database: pqsbase
mapper-location: com.njcn.influx.imapper
application:
name: oracle-data
name: oracle-influx
autoconfigure:
exclude: com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure
datasource: