Skip to content

S
sparkoffline-scala-maven-template
Commit 096ce7dd by 李辅翼
Options

v1

parents
Showing with 966 additions and 0 deletions
.idea/compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="sparkofflinescalamaventemplate" />
</profile>
</annotationProcessing>
<bytecodeTargetLevel target="8" />
</component>
</project>
\ No newline at end of file
.idea/encodings.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" addBOMForNewFiles="with NO BOM" />
</project>
\ No newline at end of file
.idea/libraries/scala_sdk_2_11_7.xml 0 → 100644
<component name="libraryTable">
<library name="scala-sdk-2.11.7" type="Scala">
<properties>
<language-level>Scala_2_11</language-level>
<compiler-classpath>
<root url="file://D:/IT-software/scala11/lib/scala-compiler.jar" />
<root url="file://D:/IT-software/scala11/lib/scala-library.jar" />
<root url="file://D:/IT-software/scala11/lib/scala-reflect.jar" />
</compiler-classpath>
</properties>
<CLASSES>
<root url="jar://D:/IT-software/scala11/lib/scala-actors-2.11.0.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-actors-migration_2.11-1.1.0.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-library.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-parser-combinators_2.11-1.0.4.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-reflect.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-swing_2.11-1.0.2.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-xml_2.11-1.0.4.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://D:/IT-software/scala11/api/jars/scala-actors-2.11.0-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-actors-migration_2.11-1.1.0-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-library-2.11.7-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-parser-combinators_2.11-1.0.4-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-reflect-2.11.7-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-swing_2.11-1.0.2-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-xml_2.11-1.0.4-javadoc.jar!/" />
</JAVADOC>
<SOURCES />
</library>
</component>
\ No newline at end of file
.idea/misc.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectInspectionProfilesVisibleTreeState">
<entry key="Project Default">
<profile-state>
<expanded-state>
<State />
<State>
<id>Class structureJava</id>
</State>
<State>
<id>Code maturityJava</id>
</State>
<State>
<id>Google Web Toolkit</id>
</State>
<State>
<id>JUnitJava</id>
</State>
<State>
<id>Java</id>
</State>
<State>
<id>Java 5Java language level migration aidsJava</id>
</State>
<State>
<id>Java 7Java language level migration aidsJava</id>
</State>
<State>
<id>Java 8Java language level migration aidsJava</id>
</State>
<State>
<id>Java language level migration aidsJava</id>
</State>
<State>
<id>JavadocJava</id>
</State>
<State>
<id>Numeric issuesJava</id>
</State>
<State>
<id>PerformanceJava</id>
</State>
<State>
<id>Spring</id>
</State>
<State>
<id>Spring AOPSpring</id>
</State>
<State>
<id>TestNGJava</id>
</State>
<State>
<id>Threading issuesJava</id>
</State>
</expanded-state>
</profile-state>
</entry>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
.idea/scala_compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaCompilerConfiguration">
<profile name="Maven 1" modules="sparkofflinescalamaventemplate" />
</component>
</project>
\ No newline at end of file
.idea/scala_settings.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaProjectSettings">
<option name="dontShowConversionDialog" value="true" />
</component>
</project>
\ No newline at end of file
.idea/uiDesigner.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>
\ No newline at end of file
README.md 0 → 100644
### 结构说明 ###
1. com.hikcreate.config<br>
配置文件目录，如：Apollo配置中心，sparkConf配置接口等
2. com.hikcreate.task<br>
业务代码，sparkStreaming处理逻辑
3. com.hikcreate.model<br>
实体类父目录
4. com.hikcreate.utils<br>
工具类父目录
5. com.hikcreate.launcher<br>
程序入口，通过传入指定参数执行相应的业务
5. lib<br>
依赖包，需要上传到hdfs集群上，路径在$SPARK-HOME$/conf/spark-defaults.xml 中指定（spark.yarn.jars）<br>
如： `spark.yarn.jars hdfs://hikbigdata/sparkJar/jars/*.jar`<br>
**注意jar包冲突**，<br>
6. script<br>
服务器脚本，主要作为一个文档管理作用，不作为执行文件，包含spark-submit脚本，hiveSql脚本
7. META-INF/app.properties<br>
Apollo配置中心地址
8. log4j.properties<br>
日志配置文件
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.hikcreate</groupId>
<artifactId>sparkoffline-scala-maven-template</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>hdp</id>
<url>https://repo.hortonworks.com/content/repositories/releases/</url>
</repository>
</repositories>
<properties>
<scala.binary.version>2.11</scala.binary.version>
<spark.version>2.4.4</spark.version>
<fastjson.version>1.2.38</fastjson.version>
<curator.version>2.9.1</curator.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>com.ctrip.framework.apollo</groupId>
<artifactId>apollo-client</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>com.ctrip.framework.apollo</groupId>
<artifactId>apollo-core</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.54</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_${scala.binary.version}</artifactId>
<version>2.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.7.3</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
</execution>
</executions>
<configuration>
<outputDirectory>lib/</outputDirectory>
<includeScope>runtime</includeScope>
<excludeTransitive>false</excludeTransitive>
</configuration>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
sparkofflinescalamaventemplate.iml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />
\ No newline at end of file
src/main/resources/hive.properties 0 → 100644
ods.table=kafka_down_connect_i,kafka_down_linktest_i,kafka_down_warn_msg_urge_todo_i,kafka_up_base_msg_vehicle_added_i,kafka_up_connect_i,kafka_up_ctrl_msg_take_travel_i,kafka_up_disconnect_i,kafka_up_disconnect_inform_i,kafka_up_exg_msg_history_location_i,kafka_up_exg_msg_real_location_i,kafka_up_exg_msg_register_i,kafka_up_exg_msg_take_ewaybill_i,kafka_up_linktest_i,kafka_up_prevention_exg_msg_device_param_i,kafka_up_prevention_exg_msg_report_driver_i,kafka_up_prevention_msg_file_complete_i,kafka_up_warn_msg_adpt_info_i,kafka_up_warn_msg_urge_todo_i,KAFKA_UNKNOWN_I
ods.special.table = kafka_base_data_display_config_i,kafka_base_into_enterprise_info_i,kafka_base_into_platform_info_i,kafka_base_into_vehicle_info_i,kafka_base_warnning_type_i
#dwd.table=dwd_base_data_display_config,dwd_base_into_enterprise_info,dwd_base_into_platform_info,dwd_base_into_vehicle_info,dwd_base_warnning_type,dwd_down_connect,dwd_down_linktest,dwd_down_warn_msg_urge_todo,dwd_up_base_msg_vehicle_added,dwd_up_connect,dwd_up_ctrl_msg_take_travel,dwd_up_disconnect,dwd_up_disconnect_inform,dwd_up_exg_msg_history_location,dwd_up_exg_msg_real_location,dwd_up_exg_msg_register,dwd_up_exg_msg_take_ewaybill,dwd_up_linktest,dwd_up_prevention_exg_msg_device_param,dwd_up_prevention_exg_msg_report_driver,dwd_up_prevention_msg_file_complete,dwd_up_warn_msg_adpt_info,dwd_up_warn_msg_urge_todo
dws.table=DWS_TRAFFIC_PART,DWS_TRAFFIC_DAY,DWS_TRAFFIC_SUM,DWS_WARN_DAY,DWS_WARN_SUM
ods.link.table = KAFKA_DOWN_LINKTEST_I,KAFKA_DOWN_CONNECT_I,KAFKA_UP_DISCONNECT_INFORM_I,KAFKA_UP_LINKTEST_I,KAFKA_UP_DISCONNECT_I,KAFKA_UP_CONNECT_I
ods.heartonline.table = KAFKA_UP_LINKTEST_I,KAFKA_DOWN_LINKTEST_I
#下级平台登录到上级平台
ods.up_connection.table = KAFKA_UP_CONNECT_I
#下级平台注销登录
ods.up_disconnect.table = KAFKA_UP_DISCONNECT_I
#主链路断开通知消息
ods.up_disconnection_inform.table = KAFKA_UP_DISCONNECT_INFORM_I
#从链路连接成功
ods.down_connection.table = KAFKA_DOWN_CONNECT_I
#上传车辆注册信息消息
ods.register.table = KAFKA_UP_EXG_MSG_REGISTER_I
#实时上传车辆定位消息和定位信息补报
ods.real_location.table = KAFKA_UP_EXG_MSG_REAL_LOCATION_I,KAFKA_UP_EXG_MSG_HISTORY_LOCATION_I
#上报车辆电子运单
ods.ewaybill.table = KAFKA_UP_EXG_MSG_TAKE_EWAYBILL_I
#报警督办请求消息和应答
ods.warn_todo.table = KAFKA_DOWN_WARN_MSG_URGE_TODO_I,KAFKA_UP_WARN_MSG_URGE_TODO_I
#上报报警信息消息
ods.warn_info.table = KAFKA_UP_WARN_MSG_ADPT_INFO_I
#上报车辆行驶记录
ods.take_travel = KAFKA_UP_CTRL_MSG_TAKE_TRAVEL_I
#补报车辆静态信息
ods.vehicle_add.table = KAFKA_UP_BASE_MSG_VEHICLE_ADDED_I
#智能视频报警设备参数查询请求应答
ods.device_param.table = KAFKA_UP_PREVENTION_EXG_MSG_DEVICE_PARAM_I
#定时上传驾驶员身份识别信息
ods.report_driver.table = KAFKA_UP_PREVENTION_EXG_MSG_REPORT_DRIVER_I
#智能视频报警附件上传结果上报
ods.file_complete.table = KAFKA_UP_PREVENTION_MSG_FILE_COMPLETE_I
\ No newline at end of file
src/main/resources/log4j.properties 0 → 100644
######################
## set log levels
######################
log4j.rootLogger=DEBUG,CONSOLE,FILEOUT
#DEBUG,CONSOLE,FILE,ROLLING_FILE,MAIL,DATABASE
log4j.addivity.org.apache=true
######################
## CONSOLE Appender ##
######################
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
log4j.appender.CONSOLE.Threshold=DEBUG
log4j.appender.CONSOLE.Target=System.out
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss,SSS}[%t] %-5p%c[%x]%X{auth}-Line:%L%m%n
######################
## Rolling File Appender
######################
log4j.appender.FILEOUT=org.apache.log4j.RollingFileAppender
log4j.appender.FILEOUT.File=E:/logs/allError.log
log4j.appender.FILEOUT.Threshold=INFO
log4j.appender.FILEOUT.Append=true
log4j.appender.fileout.MaxFileSize=1000KB
log4j.appender.ROLLING_FILE.MaxBackupIndex=5
log4j.appender.FILEOUT.layout=org.apache.log4j.PatternLayout
log4j.appender.FILEOUT.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss,SSS}[%t] %-5p%c[%x]%X{auth}-Line:%L%m%n
#######################
### File Appender
#######################
#log4j.appender.FILE=org.apache.log4j.FileAppender
#log4j.appender.FILE.File=E:/logs/all.log
#log4j.appender.FILE.Append=true
#log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
#log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p][%d{yyyy-MM-dd HH\:mm\:ss,SSS}][%c] \:%m%n
##log4j.appender.CONSOLE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n\u00A0
#####################
## Socket Appender
#####################
#log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender
#log4j.appender.SOCKET.RemoteHost=localhost
#log4j.appender.SOCKET.Port=5001
#log4j.appender.SOCKET.LocationInfo=true
## Set up for Log Facter 5
#log4j.appender.SOCKET.layout=org.apache.log4j.PatternLayout
#log4j.appender.SOCET.layout.ConversionPattern=[start]%d{DATE}[DATE]%n%p[PRIORITY]%n%x[NDC]%n%t[THREAD]%n%c[CATEGORY]%n%m[MESSAGE]%n%n
#########################
## Log Factor 5 Appender
#########################
#log4j.appender.LF5_APPENDER=org.apache.log4j.lf5.LF5Appender
#log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000
########################
## SMTP Appender
########################
#log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender
#log4j.appender.MAIL.Threshold=FATAL
#log4j.appender.MAIL.BufferSize=10
#log4j.appender.MAIL.From=openwolfl@163.com
#log4j.appender.MAIL.SMTPHost=mail.openwolf.com
#log4j.appender.MAIL.Subject=Log4J Message
#log4j.appender.MAIL.To=openwolfl@163.com
#log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout
#log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n
#########################
## JDBC Appender
########################
#log4j.appender.DATABASE=org.apache.log4j.jdbc.JDBCAppender
#log4j.appender.DATABASE.URL=jdbc:mysql://localhost:3306/test
#log4j.appender.DATABASE.driver=com.mysql.jdbc.Driver
#log4j.appender.DATABASE.user=root
#log4j.appender.DATABASE.password=root
#log4j.appender.DATABASE.sql=INSERT INTO LOG4J (Message) VALUES ('[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n')
#log4j.appender.DATABASE.layout=org.apache.log4j.PatternLayout
#log4j.appender.DATABASE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n
#log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender
#log4j.appender.A1.File=SampleMessages.log4j
#log4j.appender.A1.DatePattern=yyyyMMdd-HH'.log4j'
#log4j.appender.A1.layout=org.apache.log4j.xml.XMLLayout
####################
### myself Appender
####################
#log4j.appender.im = net.cybercorlin.util.logger.appender.IMAppender
#log4j.appender.im.host = mail.cybercorlin.net
#log4j.appender.im.username = username
#log4j.appender.im.password = password
#log4j.appender.im.recipient = corlin@yeqiangwei.com
#log4j.appender.im.layout=org.apache.log4j.PatternLayout
#log4j.appender.im.layout.ConversionPattern =[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
src/main/resources/spark.properties 0 → 100644
#spark.master = spark://104.1.67.207:7078
spark.master=local[*]
spark.appName=offline
spark.driverMemory=2g
spark.workerMemory=5g
spark.executorMemory=3g
spark.rpcMessageMaxSize=1024
src/main/scala/com/hikcreate/base/OfflineBase.scala 0 → 100644
package com.hikcreate.base
import com.hikcreate.utils.CommonUtils
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.slf4j.LoggerFactory
class OfflineBase extends Serializable {
protected val LOG = LoggerFactory.getLogger("OfflineBase")
private val sparkSession = CommonUtils.getSparkSession()
def doBussiness(): Unit = {
doTask(sparkSession)
}
/**
* 子类需要重写的业务方法
*
* @param sparkSession sparkSession
*/
protected def doTask(sparkSession: SparkSession): Unit = {
}
/**
* 将数据保存到hive表中
*
* @param dataFrame dataframe
*/
protected def saveDataToHive(dataFrame: DataFrame, mode: String, tableName: String): Unit = {
dataFrame
.write
.format("hive")
.mode(mode)
.insertInto(tableName)
}
}
src/main/scala/com/hikcreate/config/ApolloConst.scala 0 → 100644
package com.hikcreate.config
import com.ctrip.framework.apollo.{Config, ConfigService}
object ApolloConst {
val config: Config = ConfigService.getConfig("application")
//每次执行全量的时间
val frequencyTime: String = config.getProperty("frequency_date", null)
//基础信息表
val baseInfo: List[String] = List(
config.getProperty("hive.KAFKA_base_into_platform_info.table", null), //接入平台
config.getProperty("hive.KAFKA_base_into_enterprise_info.table", null), //企业信息
config.getProperty("hive.KAFKA_base_into_vehicle_info.table", null), //接入车辆
config.getProperty("hive.KAFKA_base_data_display_config.table", null), //数据展示配置
config.getProperty("hive.KAFKA_base_warning_type.table", null) //报警类型配置
)
//智能视频报警附件上传结果上报
val file_Complete_Table: String = config.getProperty("hive.UP_PREVENTION_MSG_FILE_COMPLETE.table", null)
//定时上传驾驶员身份识别信息
val report_Driver_Table: String = config.getProperty("hive.UP_PREVENTION_EXG_MSG_REPORT_DRIVER.table", null)
//视频报警设备参数查询请求
val device_Param_Table: String = config.getProperty("hive.UP_PREVENTION_EXG_MSG_DEVICE_PARAM.table", null)
//补报车辆静态信息
val vehicle_Add: String = config.getProperty("hive.UP_BASE_MSG_VEHICLE_ADDED.table", null)
//上报车辆行驶记录
val take_Travel: String = config.getProperty("hive.UP_CTRL_MSG_TAKE_TRAVEL.table", null)
//上报报警信息消息
val warn_Info_Table: String = config.getProperty("hive.UP_WARN_MSG_ADPT_INFO.table", null)
//报警督办请求消息和应答
val warn_Todo_Table: List[String] = List(
config.getProperty("hive.KAFKA_UP_WARN_MSG_URGE_TODO.table", null), //报警督办请求信息
config.getProperty("hive.KAFKA_DOWN_WARN_MSG_URGE_TODO.table", null) //报警督办应答消息
)
//上报车辆电子运单
val ewaybill_Table: String = config.getProperty("hive.UP_EXG_MSG_TAKE_EWAYBILL_ACK.table", null)
//实时上传车辆定位消息和定位信息补报
val real_Location_Table: List[String] = List(
config.getProperty("hive.UP_EXG_MSG_REAL_LOCATION.table", null), //实时上传车辆定位消息
config.getProperty("hive.KAFKA_UP_EXG_MSG_HISTORY_LOCATION.table", null) //车辆定位信息补报
)
//上传车辆注册信息消息
val register_Table: String = config.getProperty("hive.UP_EXG_MSG_REGISTER.table", null)
//从链路连接成功
val down_Connect_Table: String = config.getProperty("hive.DOWN_CONNECT.table", null)
//主链路断开
val up_Disconnection_Inform_Table: String = config.getProperty("hive.UP_DISCONNECT_INFORM.table", null)
//主链路注销
val up_Disconnect_Table: String = config.getProperty("hive.UP_DISCONNECT_REQ.table", null)
//下级平台登录到上级平台
val up_Connection_Table:String = config.getProperty("hive.UP_CONNECT_REQ.table", null)
//在线和心跳响应
val ods_Heart_Table:Array[String]=Array(
config.getProperty("hive.DOWN_LINKTEST.table", null), //从链路连接保持应答
config.getProperty("hive.UP_LINKTEST_REQ.table", null) //主链路链路保持
)
}
src/main/scala/com/hikcreate/config/HiveConfig.scala 0 → 100644
package com.hikcreate.config
import com.hikcreate.utils.CommonUtils
object HiveConfig {
//ods_table
private var ods_table = CommonUtils.getProperties("hive.properties").getProperty("ods.table")
private var ods_special_table = CommonUtils.getProperties("hive.properties").getProperty("ods.special.table")
private[this] var _ods_link_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.link.table")
private[this] var _ods_heart_online_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.heartonline.table")
private[this] var _up_connection_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.up_connection.table")
private[this] var _up_disconnect_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.up_disconnect.table")
private[this] var _up_disconnection_inform_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.up_disconnection_inform.table")
private[this] var _down_connect_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.down_connection.table")
private[this] var _register_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.register.table")
private[this] var _real_location: String = CommonUtils.getProperties("hive.properties").getProperty("ods.real_location.table")
private[this] var _ewaybill_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.ewaybill.table")
private[this] var _warn_todo_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.warn_todo.table")
private[this] var _warn_info_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.warn_info.table")
private[this] var _take_travel: String = CommonUtils.getProperties("hive.properties").getProperty("ods.take_travel")
private[this] var _vehicle_add: String = CommonUtils.getProperties("hive.properties").getProperty("ods.vehicle_add.table")
private[this] var _device_param_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.device_param.table")
private[this] var _report_driver_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.report_driver.table")
private[this] var _file_complete_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.file_complete.table")
def file_complete_table: String = _file_complete_table
def file_complete_table_=(value: String): Unit = {
_file_complete_table = value
}
def report_driver_table: String = _report_driver_table
def report_driver_table_=(value: String): Unit = {
_report_driver_table = value
}
def device_param_table: String = _device_param_table
def device_param_table_=(value: String): Unit = {
_device_param_table = value
}
def vehicle_add: String = _vehicle_add
def vehicle_add_=(value: String): Unit = {
_vehicle_add = value
}
def take_travel: String = _take_travel
def take_travel_=(value: String): Unit = {
_take_travel = value
}
def warn_info_table: String = _warn_info_table
def warn_info_table_=(value: String): Unit = {
_warn_info_table = value
}
def warn_todo_table: String = _warn_todo_table
def warn_todo_table_=(value: String): Unit = {
_warn_todo_table = value
}
def ewaybill_table: String = _ewaybill_table
def ewaybill_table_=(value: String): Unit = {
_ewaybill_table = value
}
def real_location: String = _real_location
def real_location_=(value: String): Unit = {
_real_location = value
}
def register_table: String = _register_table
def register_table_=(value: String): Unit = {
_register_table = value
}
def down_connect_table: String = _down_connect_table
def down_connect_table_=(value: String): Unit = {
_down_connect_table = value
}
def up_disconnection_inform_table: String = _up_disconnection_inform_table
def up_disconnection_inform_table_=(value: String): Unit = {
_up_disconnection_inform_table = value
}
def up_disconnect_table: String = _up_disconnect_table
def up_disconnect_table_=(value: String): Unit = {
_up_disconnect_table = value
}
def up_connection_table: String = _up_connection_table
def up_connection_table_=(value: String): Unit = {
_up_connection_table = value
}
def ods_heart_online_table: String = _ods_heart_online_table
def ods_heart_online_table_=(value: String): Unit = {
_ods_heart_online_table = value
}
def ods_link_table: String = _ods_link_table
def ods_link_table_=(value: String): Unit = {
_ods_link_table = value
}
//dwd_table
private var dwd_table = CommonUtils.getProperties("hive.properties").getProperty("dwd.table")
//dws_table
private var dws_table = CommonUtils.getProperties("hive.properties").getProperty("dws.table")
def getOds_Special_table(): String = {
ods_special_table
}
def setOds_Special_table(ods_special_table: String): Unit = {
this.ods_special_table = ods_special_table
}
def getOds_Table(): String = {
ods_table
}
def setOds_table(ods_table: String): Unit = {
this.ods_table = ods_table
}
def getDwd_Table(): String = {
dwd_table
}
def setDwd_table(dwd_table: String): Unit = {
this.dwd_table = dwd_table
}
def getDws_Table(): String = {
dws_table
}
def setDws_table(dws_table: String): Unit = {
this.dws_table = dws_table
}
}
src/main/scala/com/hikcreate/config/Logging.scala 0 → 100644
package com.hikcreate.config
import com.hikcreate.launcher.Main
import org.apache.log4j.Logger
/**
* @program: com.hikcreate.config->Logging
* @description: 日志接口
* @author: wangjiancheng
* @create: 2020/2/11 14:21
**/
//noinspection ScalaDocUnknownTag
trait Logging {
private val log = Logger.getLogger(this.getClass)
protected def debug(msg: => String): Unit = log.debug(msg)
protected def info(msg: => String): Unit = log.info(msg)
protected def warn(msg: => String): Unit = log.warn(msg)
protected def error(msg: => String): Unit = log.error(msg)
protected def error(msg: => String, exception: Exception): Unit = {
log.error(msg)
exception.printStackTrace()
}
}
src/main/scala/com/hikcreate/config/SparkConfig.scala 0 → 100644
package com.hikcreate.config
import com.hikcreate.utils.CommonUtils
/**
* spark配置类
*/
object SparkConfig {
/**
* master
*/
private var master = CommonUtils.getProperties("spark.properties").getProperty("spark.master")
/**
* appName
*/
private var appName = CommonUtils.getProperties("spark.properties").getProperty("spark.appName")
/**
* driverMemory
*/
private var driverMemory = CommonUtils.getProperties("spark.properties").getProperty("spark.driverMemory")
/**
* workerMemory
*/
private var workerMemory = CommonUtils.getProperties("spark.properties").getProperty("spark.workerMemory")
/**
* executorMemory
*/
private var executorMemory = CommonUtils.getProperties("spark.properties").getProperty("spark.executorMemory")
/**
* rpcMessageMaxSize
*/
private var rpcMessageMaxSize = CommonUtils.getProperties("spark.properties").getProperty("spark.rpcMessageMaxSize")
def getMaster() : String = {
master
}
def setMaster(master : String) : Unit = {
this.master = master
}
def getAppName() : String = {
appName
}
def setAppName(appName : String) : Unit = {
this.appName = appName
}
def getDriverMemory() : String = {
driverMemory
}
def setDriverMemory(master : String) : Unit = {
this.driverMemory = driverMemory
}
def getWorkerMemory() : String = {
workerMemory
}
def setWorkerMemory(master : String) : Unit = {
this.workerMemory = workerMemory
}
def getExecutorMemory() : String = {
executorMemory
}
def setExecutorMemory(master : String) : Unit = {
this.executorMemory = executorMemory
}
def getRpcMessageMaxSize() : String = {
rpcMessageMaxSize
}
def setRpcMessageMaxSize(master : String) : Unit = {
this.rpcMessageMaxSize = rpcMessageMaxSize
}
}
src/main/scala/com/hikcreate/launcher/Main.scala 0 → 100644
package com.hikcreate.launcher
import com.hikcreate.task.Train
import org.apache.log4j.Logger
/**
* 程序入口
*/
object Main {
//注册应用
private val logger2 = Logger.getLogger(Main.getClass)
private val FOO = Map("train" -> new Train())
def main(args: Array[String]): Unit = {
if (args == null || args.size < 1 || FOO.contains(args(0))) {
logger2.error("请指定一个启动应用：" + FOO.mkString(","))
} else {
val job = FOO(args(0))
logger2.info("开始执行任务：" + args(0))
job.doBussiness()
}
}
}
src/main/scala/com/hikcreate/task/Train.scala 0 → 100644
package com.hikcreate.task
import com.hikcreate.base.OfflineBase
import com.hikcreate.config.Logging
import com.hikcreate.utils.CommonUtils
import org.apache.spark.sql.SparkSession
class Train extends OfflineBase with Logging {
override protected def doTask(sparkSession: SparkSession): Unit = {
sparkSession.sql("set hive.exec.dynamic.partition=true")
sparkSession.sql("set hive.exec.dynamic.partition.mode=nostrict")
val DWB_safe_drive_sql ="""select * from
|table""".stripMargin
val safeTotal = sparkSession.sql(DWB_safe_drive_sql)
CommonUtils.insertDateIntoTable(safeTotal, "overwrite", "dwb_topic.dwb_safe_total")
}
}
src/main/scala/com/hikcreate/utils/DateUtils.scala 0 → 100644
package com.hikcreate.utils
import java.util.Date
import org.apache.commons.lang3.time.DateFormatUtils
object DateUtils {
private val parsePatterns = Array("yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy-MM", "yyyy/MM/dd", "yyyy/MM/dd HH:mm:ss",
"yyyy/MM/dd HH:mm", "yyyy/MM", "yyyy.MM.dd", "yyyy.MM.dd HH:mm:ss", "yyyy.MM.dd HH:mm", "yyyy.MM")
def getDate: String = DateFormatUtils.format(new Date, "yyyy-MM-dd");
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment