oralce同步到influxDB

2024-01-15 20:31:14 +08:00
parent 7b31e0376c
commit ef7f63bbec
42 changed files with 8096 additions and 1 deletions
--- a/influx-data/influx-target/.gitignore
+++ b/influx-data/influx-target/.gitignore
@@ -0,0 +1,38 @@
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### IntelliJ IDEA ###
+.idea/modules.xml
+.idea/jarRepositories.xml
+.idea/compiler.xml
+.idea/libraries/
+*.iws
+*.iml
+*.ipr
+
+### Eclipse ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
+
+### Mac OS ###
+.DS_Store
--- a/influx-data/influx-target/pom.xml
+++ b/influx-data/influx-target/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.njcn</groupId>
+        <artifactId>influx-data</artifactId>
+        <version>1.0.0</version>
+    </parent>
+
+    <artifactId>influx-target</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <!--oracle数据源-->
+        <dependency>
+            <groupId>com.njcn</groupId>
+            <artifactId>influx-source</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+
+    </dependencies>
+</project>
--- a/influx-data/influx-target/src/main/java/com/njcn/InfluxDataApplication.java
+++ b/influx-data/influx-target/src/main/java/com/njcn/InfluxDataApplication.java
@@ -0,0 +1,26 @@
+package com.njcn;
+
+import lombok.extern.slf4j.Slf4j;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.boot.autoconfigure.security.servlet.SecurityFilterAutoConfiguration;
+import org.springframework.scheduling.annotation.EnableAsync;
+
+/**
+ * pqs
+ *
+ * @author cdf
+ * @date 2022/11/10
+ */
+@Slf4j
+@MapperScan("com.njcn.**.mapper")
+@SpringBootApplication(scanBasePackages = "com.njcn",exclude = {SecurityAutoConfiguration.class, SecurityFilterAutoConfiguration.class})
+public class InfluxDataApplication {
+
+    public static void main(String[] args)  {
+        SpringApplication.run(InfluxDataApplication.class, args);
+    }
+
+}
--- a/influx-data/influx-target/src/main/java/com/njcn/influx/controller/OracleToInfluxDBController.java
+++ b/influx-data/influx-target/src/main/java/com/njcn/influx/controller/OracleToInfluxDBController.java
@@ -0,0 +1,116 @@
+package com.njcn.influx.controller;
+
+import cn.hutool.core.collection.CollectionUtil;
+import cn.hutool.core.date.LocalDateTimeUtil;
+import cn.hutool.extra.spring.SpringUtil;
+import com.njcn.influx.service.InfluxDBBaseService;
+import com.njcn.oracle.bo.param.DataAsynParam;
+import com.njcn.oracle.bo.param.MigrationParam;
+import com.njcn.oracle.mybatis.service.IReplenishMybatisService;
+import com.njcn.oracle.service.DataSyncService;
+import com.njcn.oracle.util.LocalDateUtil;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiOperation;
+import lombok.AllArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.lang.reflect.Method;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Description:
+ * Date: 2024/1/15  18:47【需求编号】
+ *
+ * @author clam
+ * @version V1.0.0
+ */
+@Validated
+@Slf4j
+@RestController
+@RequestMapping("/data")
+@Api(tags = "OracleToInfluxDB数据同步")
+@AllArgsConstructor
+public class OracleToInfluxDBController {
+
+    private final InfluxDBBaseService influxDBBaseService;
+    private final DataSyncService dataSyncService;
+    @PostMapping("/dataSync")
+    @ApiOperation("数据同步")
+    @ApiImplicitParam(name = "dataAsynParam", value = "数据同步参数", required = true)
+    @SneakyThrows
+    public Boolean dataSync(@RequestBody DataAsynParam dataAsynParam){
+        dataAsynParam.getTableNames().stream().forEach(temp->{
+            IReplenishMybatisService executor = null;
+            try {
+                executor = (IReplenishMybatisService) SpringUtil.getBean(Class.forName("com.njcn.oracle.service.impl." + temp + "ServiceImpl"));
+            } catch (ClassNotFoundException e) {
+                throw new RuntimeException(e);
+            }
+
+            IReplenishMybatisService finalExecutor = executor;
+            MigrationParam migration = new MigrationParam();
+
+            LocalDateTime localDateTime = dataAsynParam.getStartTime().atStartOfDay();
+            LocalDateTime tempStartTime = LocalDateTimeUtil.beginOfDay(localDateTime);
+            LocalDateTime tempEndTime = LocalDateTimeUtil.endOfDay(localDateTime);
+
+            migration.setStartTime(tempStartTime);
+            migration.setEndTime(tempEndTime);
+            migration.setStartTime(tempStartTime);
+            migration.setEndTime(tempEndTime);
+            List list = finalExecutor.queryData(migration);
+            if (CollectionUtil.isEmpty(list)) {
+                return;
+            }
+            Class<?> clazz = null;
+            Class<?> clazz2 = null;
+
+            try {
+                clazz = Class.forName("com.njcn.influx.bo.po.InfluxDB" + temp);
+                clazz2 = Class.forName("com.njcn.oracle.bo.po." + temp);
+            } catch (ClassNotFoundException e) {
+                throw new RuntimeException(e);
+            }
+            Method method = null;
+            try {
+                method = clazz.getDeclaredMethod("oralceToInfluxDB",clazz2);
+            } catch (NoSuchMethodException e) {
+                throw new RuntimeException(e);
+            }
+            method.setAccessible(true);
+            Method finalMethod = method;
+            List list1 =(List) list.stream().flatMap(po -> {
+                try {
+
+                    Object invoke = finalMethod.invoke(null,po);
+                    Object invoke1 = invoke;
+                    //返回可能是集合
+                    return invoke1 instanceof List ? ((List<?>) invoke1).stream() : Stream.of(invoke1);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+
+            }).map(item-> (Object) item).collect(Collectors.toList());
+
+            influxDBBaseService.insertBatchBySlice(list1,5000);
+
+
+        });
+
+        return  true;// HttpResultUtil.assembleCommonResponseResult(CommonResponseEnum.SUCCESS, true, "数据同步");
+    }
+
+
+}
--- a/influx-data/influx-target/src/main/resources/application.yml
+++ b/influx-data/influx-target/src/main/resources/application.yml
@@ -0,0 +1,111 @@
+#文件位置配置
+business:
+  #处理波形数据位置
+  # wavePath: C://comtrade
+  wavePath: /usr/local/comtrade
+  #处理临时数据
+  #tempPath: C://file
+  tempPath: /usr/local/file
+  #文件存储的方式
+  file:
+    storage: 3
+microservice:
+  ename: @artifactId@
+  name: oracle-data
+#线程池配置信息
+threadPool:
+  corePoolSize: 10
+  maxPoolSize: 20
+  queueCapacity: 500
+  keepAliveSeconds: 60
+server:
+  port: 8090
+spring:
+  #influxDB内容配置
+  influx:
+    url: http://192.168.1.102:8086
+    user: admin
+    password: 123456
+    database: pqsbase_zl
+    mapper-location: com.njcn.influx.imapper
+  application:
+    name: oracle-data
+  autoconfigure:
+    exclude: com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure
+  datasource:
+    dynamic:
+      druid:
+        initial-size: 10
+        # 初始化大小，最小，最大
+        min-idle: 20
+        maxActive: 500
+        # 配置获取连接等待超时的时间
+        maxWait: 60000
+        # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
+        timeBetweenEvictionRunsMillis: 60000
+        # 配置一个连接在池中最小生存的时间，单位是毫秒
+        minEvictableIdleTimeMillis: 300000
+        testWhileIdle: true
+        testOnBorrow: true
+        validation-query: SELECT 1 from dual
+        testOnReturn: false
+        # 打开PSCache，并且指定每个连接上PSCache的大小
+        poolPreparedStatements: true
+        maxPoolPreparedStatementPerConnectionSize: 20
+        filters: stat,wall
+        filter:
+          wall:
+            config:
+              multi-statement-allow: true
+              none-base-statement-allow: true
+            enabled: true
+        # 配置DruidStatFilter
+        web-stat-filter:
+          enabled: true
+          url-pattern: "/*"
+          exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*"
+        # 配置DruidStatViewServlet
+        stat-view-servlet:
+          enabled: true
+          url-pattern: "/druid/*"
+          # IP白名单(没有配置或者为空，则允许所有访问)
+          allow: #127.0.0.1,192.168.163.1
+          # IP黑名单 (存在共同时，deny优先于allow)
+          deny: #192.168.1.73
+          #  禁用HTML页面上的“Reset All”功能
+          reset-enable: false
+          # 登录名
+          login-username: admin
+          # 登录密码
+          login-password: njcnpqs
+        query-timeout: 36000
+      primary: master
+      strict: false
+      datasource:
+        master:
+          url: jdbc:oracle:thin:@192.168.1.51:1521:pqsbase
+          username: pqsadmin
+          password: Pqsadmin123
+          driver-class-name: oracle.jdbc.driver.OracleDriver
+        target:
+          url: jdbc:oracle:thin:@192.168.1.51:1521:pqsbase
+          username: pqsadmin_hn
+          password: pqsadmin
+          driver-class-name: oracle.jdbc.driver.OracleDriver
+#mybatis配置信息
+mybatis-plus:
+  #别名扫描
+  type-aliases-package: com.njcn.oracle.bo
+  mapper-locations: classpath*:com/njcn/**/mapping/*.xml
+  configuration:
+    #驼峰命名
+    map-underscore-to-camel-case: true
+    #配置sql日志输出
+#    log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
+    #关闭日志输出
+    log-impl: org.apache.ibatis.logging.nologging.NoLoggingImpl
+  global-config:
+    db-config:
+      #指定主键生成策略
+      id-type: assign_uuid
+
--- a/influx-data/influx-target/src/test/java/njcn/BaseJunitTest.java
+++ b/influx-data/influx-target/src/test/java/njcn/BaseJunitTest.java
@@ -0,0 +1,18 @@
+package njcn;
+
+import com.njcn.InfluxDataApplication;
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.web.WebAppConfiguration;
+
+/**
+ * @author hongawen
+ * @version 1.0.0
+ * @date 2021年12月10日 15:05
+ */
+@RunWith(SpringRunner.class)
+@WebAppConfiguration
+@SpringBootTest(classes = InfluxDataApplication.class)
+public class BaseJunitTest {
+}
--- a/influx-data/influx-target/src/test/java/njcn/DataTest.java
+++ b/influx-data/influx-target/src/test/java/njcn/DataTest.java
@@ -0,0 +1,64 @@
+package njcn;
+
+
+import cn.hutool.core.date.DatePattern;
+import cn.hutool.core.date.LocalDateTimeUtil;
+import cn.hutool.extra.spring.SpringUtil;
+import com.njcn.influx.bo.po.InfluxDBDataFlicker;
+import com.njcn.influx.core.InfluxExecutor;
+import com.njcn.oracle.bo.param.MigrationParam;
+import com.njcn.oracle.bo.po.DataFlicker;
+import com.njcn.oracle.mybatis.service.IReplenishMybatisService;
+import lombok.SneakyThrows;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author hongawen
+ * @version 1.0.0
+ * @date 2021年12月14日 12:55
+ */
+public class DataTest extends BaseJunitTest {
+    private @Autowired InfluxExecutor  interpreter;
+
+
+
+    @SneakyThrows
+    @Test
+    public void testBean() {
+        LocalDateTime endTime = LocalDateTimeUtil.parse("2023-12-29 15:00:10", DatePattern.NORM_DATETIME_PATTERN);
+        LocalDateTime startTime = LocalDateTimeUtil.offset(endTime, -24, ChronoUnit.HOURS);
+
+//        LocalDateTime endTime = LocalDateTimeUtil.parse("1970-01-02 00:00:10", DatePattern.NORM_DATETIME_PATTERN);
+//        LocalDateTime startTime = LocalDateTimeUtil.offset(endTime, -2, ChronoUnit.HOURS);
+
+        MigrationParam migrationParam = new MigrationParam();
+        migrationParam.setStartTime(startTime);
+        migrationParam.setEndTime(endTime);
+        IReplenishMybatisService executor = (IReplenishMybatisService) SpringUtil.getBean(Class.forName("com.njcn.oracle.service.impl.DataFlickerServiceImpl"));
+        //查询数据
+        List<DataFlicker> temp1 = executor.queryData(migrationParam);
+        List<InfluxDBDataFlicker> collect = temp1.stream().map(temp -> {
+            InfluxDBDataFlicker dataFlicker = new InfluxDBDataFlicker();
+            dataFlicker = InfluxDBDataFlicker.oralceToInfluxDB(temp);
+
+            return dataFlicker;
+        }).collect(Collectors.toList());
+
+        Object args[] ={collect};
+        interpreter.insert(args);
+        System.out.println(1111111);
+
+    }
+
+
+
+}