Skip to content

S
sparkoffline-scala-maven-template
Commit 096ce7dd by 李辅翼
Options

v1

parents
Showing with 1387 additions and 0 deletions
.idea/compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="sparkofflinescalamaventemplate" />
</profile>
</annotationProcessing>
<bytecodeTargetLevel target="8" />
</component>
</project>
\ No newline at end of file
.idea/encodings.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" addBOMForNewFiles="with NO BOM" />
</project>
\ No newline at end of file
.idea/libraries/scala_sdk_2_11_7.xml 0 → 100644
<component name="libraryTable">
<library name="scala-sdk-2.11.7" type="Scala">
<properties>
<language-level>Scala_2_11</language-level>
<compiler-classpath>
<root url="file://D:/IT-software/scala11/lib/scala-compiler.jar" />
<root url="file://D:/IT-software/scala11/lib/scala-library.jar" />
<root url="file://D:/IT-software/scala11/lib/scala-reflect.jar" />
</compiler-classpath>
</properties>
<CLASSES>
<root url="jar://D:/IT-software/scala11/lib/scala-actors-2.11.0.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-actors-migration_2.11-1.1.0.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-library.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-parser-combinators_2.11-1.0.4.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-reflect.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-swing_2.11-1.0.2.jar!/" />
<root url="jar://D:/IT-software/scala11/lib/scala-xml_2.11-1.0.4.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://D:/IT-software/scala11/api/jars/scala-actors-2.11.0-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-actors-migration_2.11-1.1.0-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-library-2.11.7-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-parser-combinators_2.11-1.0.4-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-reflect-2.11.7-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-swing_2.11-1.0.2-javadoc.jar!/" />
<root url="jar://D:/IT-software/scala11/api/jars/scala-xml_2.11-1.0.4-javadoc.jar!/" />
</JAVADOC>
<SOURCES />
</library>
</component>
\ No newline at end of file
.idea/misc.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectInspectionProfilesVisibleTreeState">
<entry key="Project Default">
<profile-state>
<expanded-state>
<State />
<State>
<id>Class structureJava</id>
</State>
<State>
<id>Code maturityJava</id>
</State>
<State>
<id>Google Web Toolkit</id>
</State>
<State>
<id>JUnitJava</id>
</State>
<State>
<id>Java</id>
</State>
<State>
<id>Java 5Java language level migration aidsJava</id>
</State>
<State>
<id>Java 7Java language level migration aidsJava</id>
</State>
<State>
<id>Java 8Java language level migration aidsJava</id>
</State>
<State>
<id>Java language level migration aidsJava</id>
</State>
<State>
<id>JavadocJava</id>
</State>
<State>
<id>Numeric issuesJava</id>
</State>
<State>
<id>PerformanceJava</id>
</State>
<State>
<id>Spring</id>
</State>
<State>
<id>Spring AOPSpring</id>
</State>
<State>
<id>TestNGJava</id>
</State>
<State>
<id>Threading issuesJava</id>
</State>
</expanded-state>
</profile-state>
</entry>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
.idea/scala_compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaCompilerConfiguration">
<profile name="Maven 1" modules="sparkofflinescalamaventemplate" />
</component>
</project>
\ No newline at end of file
.idea/scala_settings.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaProjectSettings">
<option name="dontShowConversionDialog" value="true" />
</component>
</project>
\ No newline at end of file
.idea/uiDesigner.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>
\ No newline at end of file
README.md 0 → 100644
### 结构说明 ###
1. com.hikcreate.config<br>
配置文件目录，如：Apollo配置中心，sparkConf配置接口等
2. com.hikcreate.task<br>
业务代码，sparkStreaming处理逻辑
3. com.hikcreate.model<br>
实体类父目录
4. com.hikcreate.utils<br>
工具类父目录
5. com.hikcreate.launcher<br>
程序入口，通过传入指定参数执行相应的业务
5. lib<br>
依赖包，需要上传到hdfs集群上，路径在$SPARK-HOME$/conf/spark-defaults.xml 中指定（spark.yarn.jars）<br>
如： `spark.yarn.jars hdfs://hikbigdata/sparkJar/jars/*.jar`<br>
**注意jar包冲突**，<br>
6. script<br>
服务器脚本，主要作为一个文档管理作用，不作为执行文件，包含spark-submit脚本，hiveSql脚本
7. META-INF/app.properties<br>
Apollo配置中心地址
8. log4j.properties<br>
日志配置文件
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.hikcreate</groupId>
<artifactId>sparkoffline-scala-maven-template</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>hdp</id>
<url>https://repo.hortonworks.com/content/repositories/releases/</url>
</repository>
</repositories>
<properties>
<scala.binary.version>2.11</scala.binary.version>
<spark.version>2.4.4</spark.version>
<fastjson.version>1.2.38</fastjson.version>
<curator.version>2.9.1</curator.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>com.ctrip.framework.apollo</groupId>
<artifactId>apollo-client</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>com.ctrip.framework.apollo</groupId>
<artifactId>apollo-core</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<!--<scope>provided</scope>-->
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.54</version>
</dependency>
<dependency>
<groupId>org.scalaj</groupId>
<artifactId>scalaj-http_${scala.binary.version}</artifactId>
<version>2.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.7.3</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
</execution>
</executions>
<configuration>
<outputDirectory>lib/</outputDirectory>
<includeScope>runtime</includeScope>
<excludeTransitive>false</excludeTransitive>
</configuration>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
sparkofflinescalamaventemplate.iml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />
\ No newline at end of file
src/main/resources/hive.properties 0 → 100644
ods.table=kafka_down_connect_i,kafka_down_linktest_i,kafka_down_warn_msg_urge_todo_i,kafka_up_base_msg_vehicle_added_i,kafka_up_connect_i,kafka_up_ctrl_msg_take_travel_i,kafka_up_disconnect_i,kafka_up_disconnect_inform_i,kafka_up_exg_msg_history_location_i,kafka_up_exg_msg_real_location_i,kafka_up_exg_msg_register_i,kafka_up_exg_msg_take_ewaybill_i,kafka_up_linktest_i,kafka_up_prevention_exg_msg_device_param_i,kafka_up_prevention_exg_msg_report_driver_i,kafka_up_prevention_msg_file_complete_i,kafka_up_warn_msg_adpt_info_i,kafka_up_warn_msg_urge_todo_i,KAFKA_UNKNOWN_I
ods.special.table = kafka_base_data_display_config_i,kafka_base_into_enterprise_info_i,kafka_base_into_platform_info_i,kafka_base_into_vehicle_info_i,kafka_base_warnning_type_i
#dwd.table=dwd_base_data_display_config,dwd_base_into_enterprise_info,dwd_base_into_platform_info,dwd_base_into_vehicle_info,dwd_base_warnning_type,dwd_down_connect,dwd_down_linktest,dwd_down_warn_msg_urge_todo,dwd_up_base_msg_vehicle_added,dwd_up_connect,dwd_up_ctrl_msg_take_travel,dwd_up_disconnect,dwd_up_disconnect_inform,dwd_up_exg_msg_history_location,dwd_up_exg_msg_real_location,dwd_up_exg_msg_register,dwd_up_exg_msg_take_ewaybill,dwd_up_linktest,dwd_up_prevention_exg_msg_device_param,dwd_up_prevention_exg_msg_report_driver,dwd_up_prevention_msg_file_complete,dwd_up_warn_msg_adpt_info,dwd_up_warn_msg_urge_todo
dws.table=DWS_TRAFFIC_PART,DWS_TRAFFIC_DAY,DWS_TRAFFIC_SUM,DWS_WARN_DAY,DWS_WARN_SUM
ods.link.table = KAFKA_DOWN_LINKTEST_I,KAFKA_DOWN_CONNECT_I,KAFKA_UP_DISCONNECT_INFORM_I,KAFKA_UP_LINKTEST_I,KAFKA_UP_DISCONNECT_I,KAFKA_UP_CONNECT_I
ods.heartonline.table = KAFKA_UP_LINKTEST_I,KAFKA_DOWN_LINKTEST_I
#下级平台登录到上级平台
ods.up_connection.table = KAFKA_UP_CONNECT_I
#下级平台注销登录
ods.up_disconnect.table = KAFKA_UP_DISCONNECT_I
#主链路断开通知消息
ods.up_disconnection_inform.table = KAFKA_UP_DISCONNECT_INFORM_I
#从链路连接成功
ods.down_connection.table = KAFKA_DOWN_CONNECT_I
#上传车辆注册信息消息
ods.register.table = KAFKA_UP_EXG_MSG_REGISTER_I
#实时上传车辆定位消息和定位信息补报
ods.real_location.table = KAFKA_UP_EXG_MSG_REAL_LOCATION_I,KAFKA_UP_EXG_MSG_HISTORY_LOCATION_I
#上报车辆电子运单
ods.ewaybill.table = KAFKA_UP_EXG_MSG_TAKE_EWAYBILL_I
#报警督办请求消息和应答
ods.warn_todo.table = KAFKA_DOWN_WARN_MSG_URGE_TODO_I,KAFKA_UP_WARN_MSG_URGE_TODO_I
#上报报警信息消息
ods.warn_info.table = KAFKA_UP_WARN_MSG_ADPT_INFO_I
#上报车辆行驶记录
ods.take_travel = KAFKA_UP_CTRL_MSG_TAKE_TRAVEL_I
#补报车辆静态信息
ods.vehicle_add.table = KAFKA_UP_BASE_MSG_VEHICLE_ADDED_I
#智能视频报警设备参数查询请求应答
ods.device_param.table = KAFKA_UP_PREVENTION_EXG_MSG_DEVICE_PARAM_I
#定时上传驾驶员身份识别信息
ods.report_driver.table = KAFKA_UP_PREVENTION_EXG_MSG_REPORT_DRIVER_I
#智能视频报警附件上传结果上报
ods.file_complete.table = KAFKA_UP_PREVENTION_MSG_FILE_COMPLETE_I
\ No newline at end of file
src/main/resources/log4j.properties 0 → 100644
######################
## set log levels
######################
log4j.rootLogger=DEBUG,CONSOLE,FILEOUT
#DEBUG,CONSOLE,FILE,ROLLING_FILE,MAIL,DATABASE
log4j.addivity.org.apache=true
######################
## CONSOLE Appender ##
######################
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
log4j.appender.CONSOLE.Threshold=DEBUG
log4j.appender.CONSOLE.Target=System.out
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss,SSS}[%t] %-5p%c[%x]%X{auth}-Line:%L%m%n
######################
## Rolling File Appender
######################
log4j.appender.FILEOUT=org.apache.log4j.RollingFileAppender
log4j.appender.FILEOUT.File=E:/logs/allError.log
log4j.appender.FILEOUT.Threshold=INFO
log4j.appender.FILEOUT.Append=true
log4j.appender.fileout.MaxFileSize=1000KB
log4j.appender.ROLLING_FILE.MaxBackupIndex=5
log4j.appender.FILEOUT.layout=org.apache.log4j.PatternLayout
log4j.appender.FILEOUT.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss,SSS}[%t] %-5p%c[%x]%X{auth}-Line:%L%m%n
#######################
### File Appender
#######################
#log4j.appender.FILE=org.apache.log4j.FileAppender
#log4j.appender.FILE.File=E:/logs/all.log
#log4j.appender.FILE.Append=true
#log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
#log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p][%d{yyyy-MM-dd HH\:mm\:ss,SSS}][%c] \:%m%n
##log4j.appender.CONSOLE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n\u00A0
#####################
## Socket Appender
#####################
#log4j.appender.SOCKET=org.apache.log4j.RollingFileAppender
#log4j.appender.SOCKET.RemoteHost=localhost
#log4j.appender.SOCKET.Port=5001
#log4j.appender.SOCKET.LocationInfo=true
## Set up for Log Facter 5
#log4j.appender.SOCKET.layout=org.apache.log4j.PatternLayout
#log4j.appender.SOCET.layout.ConversionPattern=[start]%d{DATE}[DATE]%n%p[PRIORITY]%n%x[NDC]%n%t[THREAD]%n%c[CATEGORY]%n%m[MESSAGE]%n%n
#########################
## Log Factor 5 Appender
#########################
#log4j.appender.LF5_APPENDER=org.apache.log4j.lf5.LF5Appender
#log4j.appender.LF5_APPENDER.MaxNumberOfRecords=2000
########################
## SMTP Appender
########################
#log4j.appender.MAIL=org.apache.log4j.net.SMTPAppender
#log4j.appender.MAIL.Threshold=FATAL
#log4j.appender.MAIL.BufferSize=10
#log4j.appender.MAIL.From=openwolfl@163.com
#log4j.appender.MAIL.SMTPHost=mail.openwolf.com
#log4j.appender.MAIL.Subject=Log4J Message
#log4j.appender.MAIL.To=openwolfl@163.com
#log4j.appender.MAIL.layout=org.apache.log4j.PatternLayout
#log4j.appender.MAIL.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n
#########################
## JDBC Appender
########################
#log4j.appender.DATABASE=org.apache.log4j.jdbc.JDBCAppender
#log4j.appender.DATABASE.URL=jdbc:mysql://localhost:3306/test
#log4j.appender.DATABASE.driver=com.mysql.jdbc.Driver
#log4j.appender.DATABASE.user=root
#log4j.appender.DATABASE.password=root
#log4j.appender.DATABASE.sql=INSERT INTO LOG4J (Message) VALUES ('[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n')
#log4j.appender.DATABASE.layout=org.apache.log4j.PatternLayout
#log4j.appender.DATABASE.layout.ConversionPattern=[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n
#log4j.appender.A1=org.apache.log4j.DailyRollingFileAppender
#log4j.appender.A1.File=SampleMessages.log4j
#log4j.appender.A1.DatePattern=yyyyMMdd-HH'.log4j'
#log4j.appender.A1.layout=org.apache.log4j.xml.XMLLayout
####################
### myself Appender
####################
#log4j.appender.im = net.cybercorlin.util.logger.appender.IMAppender
#log4j.appender.im.host = mail.cybercorlin.net
#log4j.appender.im.username = username
#log4j.appender.im.password = password
#log4j.appender.im.recipient = corlin@yeqiangwei.com
#log4j.appender.im.layout=org.apache.log4j.PatternLayout
#log4j.appender.im.layout.ConversionPattern =[framework] %d - %c -%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
src/main/resources/spark.properties 0 → 100644
#spark.master = spark://104.1.67.207:7078
spark.master=local[*]
spark.appName=offline
spark.driverMemory=2g
spark.workerMemory=5g
spark.executorMemory=3g
spark.rpcMessageMaxSize=1024
src/main/scala/com/hikcreate/base/OfflineBase.scala 0 → 100644
package com.hikcreate.base
import com.hikcreate.utils.CommonUtils
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.slf4j.LoggerFactory
class OfflineBase extends Serializable {
protected val LOG = LoggerFactory.getLogger("OfflineBase")
private val sparkSession = CommonUtils.getSparkSession()
def doBussiness(): Unit = {
doTask(sparkSession)
}
/**
* 子类需要重写的业务方法
*
* @param sparkSession sparkSession
*/
protected def doTask(sparkSession: SparkSession): Unit = {
}
/**
* 将数据保存到hive表中
*
* @param dataFrame dataframe
*/
protected def saveDataToHive(dataFrame: DataFrame, mode: String, tableName: String): Unit = {
dataFrame
.write
.format("hive")
.mode(mode)
.insertInto(tableName)
}
}
src/main/scala/com/hikcreate/config/ApolloConst.scala 0 → 100644
package com.hikcreate.config
import com.ctrip.framework.apollo.{Config, ConfigService}
object ApolloConst {
val config: Config = ConfigService.getConfig("application")
//每次执行全量的时间
val frequencyTime: String = config.getProperty("frequency_date", null)
//基础信息表
val baseInfo: List[String] = List(
config.getProperty("hive.KAFKA_base_into_platform_info.table", null), //接入平台
config.getProperty("hive.KAFKA_base_into_enterprise_info.table", null), //企业信息
config.getProperty("hive.KAFKA_base_into_vehicle_info.table", null), //接入车辆
config.getProperty("hive.KAFKA_base_data_display_config.table", null), //数据展示配置
config.getProperty("hive.KAFKA_base_warning_type.table", null) //报警类型配置
)
//智能视频报警附件上传结果上报
val file_Complete_Table: String = config.getProperty("hive.UP_PREVENTION_MSG_FILE_COMPLETE.table", null)
//定时上传驾驶员身份识别信息
val report_Driver_Table: String = config.getProperty("hive.UP_PREVENTION_EXG_MSG_REPORT_DRIVER.table", null)
//视频报警设备参数查询请求
val device_Param_Table: String = config.getProperty("hive.UP_PREVENTION_EXG_MSG_DEVICE_PARAM.table", null)
//补报车辆静态信息
val vehicle_Add: String = config.getProperty("hive.UP_BASE_MSG_VEHICLE_ADDED.table", null)
//上报车辆行驶记录
val take_Travel: String = config.getProperty("hive.UP_CTRL_MSG_TAKE_TRAVEL.table", null)
//上报报警信息消息
val warn_Info_Table: String = config.getProperty("hive.UP_WARN_MSG_ADPT_INFO.table", null)
//报警督办请求消息和应答
val warn_Todo_Table: List[String] = List(
config.getProperty("hive.KAFKA_UP_WARN_MSG_URGE_TODO.table", null), //报警督办请求信息
config.getProperty("hive.KAFKA_DOWN_WARN_MSG_URGE_TODO.table", null) //报警督办应答消息
)
//上报车辆电子运单
val ewaybill_Table: String = config.getProperty("hive.UP_EXG_MSG_TAKE_EWAYBILL_ACK.table", null)
//实时上传车辆定位消息和定位信息补报
val real_Location_Table: List[String] = List(
config.getProperty("hive.UP_EXG_MSG_REAL_LOCATION.table", null), //实时上传车辆定位消息
config.getProperty("hive.KAFKA_UP_EXG_MSG_HISTORY_LOCATION.table", null) //车辆定位信息补报
)
//上传车辆注册信息消息
val register_Table: String = config.getProperty("hive.UP_EXG_MSG_REGISTER.table", null)
//从链路连接成功
val down_Connect_Table: String = config.getProperty("hive.DOWN_CONNECT.table", null)
//主链路断开
val up_Disconnection_Inform_Table: String = config.getProperty("hive.UP_DISCONNECT_INFORM.table", null)
//主链路注销
val up_Disconnect_Table: String = config.getProperty("hive.UP_DISCONNECT_REQ.table", null)
//下级平台登录到上级平台
val up_Connection_Table:String = config.getProperty("hive.UP_CONNECT_REQ.table", null)
//在线和心跳响应
val ods_Heart_Table:Array[String]=Array(
config.getProperty("hive.DOWN_LINKTEST.table", null), //从链路连接保持应答
config.getProperty("hive.UP_LINKTEST_REQ.table", null) //主链路链路保持
)
}
src/main/scala/com/hikcreate/config/HiveConfig.scala 0 → 100644
package com.hikcreate.config
import com.hikcreate.utils.CommonUtils
object HiveConfig {
//ods_table
private var ods_table = CommonUtils.getProperties("hive.properties").getProperty("ods.table")
private var ods_special_table = CommonUtils.getProperties("hive.properties").getProperty("ods.special.table")
private[this] var _ods_link_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.link.table")
private[this] var _ods_heart_online_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.heartonline.table")
private[this] var _up_connection_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.up_connection.table")
private[this] var _up_disconnect_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.up_disconnect.table")
private[this] var _up_disconnection_inform_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.up_disconnection_inform.table")
private[this] var _down_connect_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.down_connection.table")
private[this] var _register_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.register.table")
private[this] var _real_location: String = CommonUtils.getProperties("hive.properties").getProperty("ods.real_location.table")
private[this] var _ewaybill_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.ewaybill.table")
private[this] var _warn_todo_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.warn_todo.table")
private[this] var _warn_info_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.warn_info.table")
private[this] var _take_travel: String = CommonUtils.getProperties("hive.properties").getProperty("ods.take_travel")
private[this] var _vehicle_add: String = CommonUtils.getProperties("hive.properties").getProperty("ods.vehicle_add.table")
private[this] var _device_param_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.device_param.table")
private[this] var _report_driver_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.report_driver.table")
private[this] var _file_complete_table: String = CommonUtils.getProperties("hive.properties").getProperty("ods.file_complete.table")
def file_complete_table: String = _file_complete_table
def file_complete_table_=(value: String): Unit = {
_file_complete_table = value
}
def report_driver_table: String = _report_driver_table
def report_driver_table_=(value: String): Unit = {
_report_driver_table = value
}
def device_param_table: String = _device_param_table
def device_param_table_=(value: String): Unit = {
_device_param_table = value
}
def vehicle_add: String = _vehicle_add
def vehicle_add_=(value: String): Unit = {
_vehicle_add = value
}
def take_travel: String = _take_travel
def take_travel_=(value: String): Unit = {
_take_travel = value
}
def warn_info_table: String = _warn_info_table
def warn_info_table_=(value: String): Unit = {
_warn_info_table = value
}
def warn_todo_table: String = _warn_todo_table
def warn_todo_table_=(value: String): Unit = {
_warn_todo_table = value
}
def ewaybill_table: String = _ewaybill_table
def ewaybill_table_=(value: String): Unit = {
_ewaybill_table = value
}
def real_location: String = _real_location
def real_location_=(value: String): Unit = {
_real_location = value
}
def register_table: String = _register_table
def register_table_=(value: String): Unit = {
_register_table = value
}
def down_connect_table: String = _down_connect_table
def down_connect_table_=(value: String): Unit = {
_down_connect_table = value
}
def up_disconnection_inform_table: String = _up_disconnection_inform_table
def up_disconnection_inform_table_=(value: String): Unit = {
_up_disconnection_inform_table = value
}
def up_disconnect_table: String = _up_disconnect_table
def up_disconnect_table_=(value: String): Unit = {
_up_disconnect_table = value
}
def up_connection_table: String = _up_connection_table
def up_connection_table_=(value: String): Unit = {
_up_connection_table = value
}
def ods_heart_online_table: String = _ods_heart_online_table
def ods_heart_online_table_=(value: String): Unit = {
_ods_heart_online_table = value
}
def ods_link_table: String = _ods_link_table
def ods_link_table_=(value: String): Unit = {
_ods_link_table = value
}
//dwd_table
private var dwd_table = CommonUtils.getProperties("hive.properties").getProperty("dwd.table")
//dws_table
private var dws_table = CommonUtils.getProperties("hive.properties").getProperty("dws.table")
def getOds_Special_table(): String = {
ods_special_table
}
def setOds_Special_table(ods_special_table: String): Unit = {
this.ods_special_table = ods_special_table
}
def getOds_Table(): String = {
ods_table
}
def setOds_table(ods_table: String): Unit = {
this.ods_table = ods_table
}
def getDwd_Table(): String = {
dwd_table
}
def setDwd_table(dwd_table: String): Unit = {
this.dwd_table = dwd_table
}
def getDws_Table(): String = {
dws_table
}
def setDws_table(dws_table: String): Unit = {
this.dws_table = dws_table
}
}
src/main/scala/com/hikcreate/config/Logging.scala 0 → 100644
package com.hikcreate.config
import com.hikcreate.launcher.Main
import org.apache.log4j.Logger
/**
* @program: com.hikcreate.config->Logging
* @description: 日志接口
* @author: wangjiancheng
* @create: 2020/2/11 14:21
**/
//noinspection ScalaDocUnknownTag
trait Logging {
private val log = Logger.getLogger(this.getClass)
protected def debug(msg: => String): Unit = log.debug(msg)
protected def info(msg: => String): Unit = log.info(msg)
protected def warn(msg: => String): Unit = log.warn(msg)
protected def error(msg: => String): Unit = log.error(msg)
protected def error(msg: => String, exception: Exception): Unit = {
log.error(msg)
exception.printStackTrace()
}
}
src/main/scala/com/hikcreate/config/SparkConfig.scala 0 → 100644
package com.hikcreate.config
import com.hikcreate.utils.CommonUtils
/**
* spark配置类
*/
object SparkConfig {
/**
* master
*/
private var master = CommonUtils.getProperties("spark.properties").getProperty("spark.master")
/**
* appName
*/
private var appName = CommonUtils.getProperties("spark.properties").getProperty("spark.appName")
/**
* driverMemory
*/
private var driverMemory = CommonUtils.getProperties("spark.properties").getProperty("spark.driverMemory")
/**
* workerMemory
*/
private var workerMemory = CommonUtils.getProperties("spark.properties").getProperty("spark.workerMemory")
/**
* executorMemory
*/
private var executorMemory = CommonUtils.getProperties("spark.properties").getProperty("spark.executorMemory")
/**
* rpcMessageMaxSize
*/
private var rpcMessageMaxSize = CommonUtils.getProperties("spark.properties").getProperty("spark.rpcMessageMaxSize")
def getMaster() : String = {
master
}
def setMaster(master : String) : Unit = {
this.master = master
}
def getAppName() : String = {
appName
}
def setAppName(appName : String) : Unit = {
this.appName = appName
}
def getDriverMemory() : String = {
driverMemory
}
def setDriverMemory(master : String) : Unit = {
this.driverMemory = driverMemory
}
def getWorkerMemory() : String = {
workerMemory
}
def setWorkerMemory(master : String) : Unit = {
this.workerMemory = workerMemory
}
def getExecutorMemory() : String = {
executorMemory
}
def setExecutorMemory(master : String) : Unit = {
this.executorMemory = executorMemory
}
def getRpcMessageMaxSize() : String = {
rpcMessageMaxSize
}
def setRpcMessageMaxSize(master : String) : Unit = {
this.rpcMessageMaxSize = rpcMessageMaxSize
}
}
src/main/scala/com/hikcreate/launcher/Main.scala 0 → 100644
package com.hikcreate.launcher
import com.hikcreate.task.Train
import org.apache.log4j.Logger
/**
* 程序入口
*/
object Main {
//注册应用
private val logger2 = Logger.getLogger(Main.getClass)
private val FOO = Map("train" -> new Train())
def main(args: Array[String]): Unit = {
if (args == null || args.size < 1 || FOO.contains(args(0))) {
logger2.error("请指定一个启动应用：" + FOO.mkString(","))
} else {
val job = FOO(args(0))
logger2.info("开始执行任务：" + args(0))
job.doBussiness()
}
}
}
src/main/scala/com/hikcreate/task/Train.scala 0 → 100644
package com.hikcreate.task
import com.hikcreate.base.OfflineBase
import com.hikcreate.config.Logging
import com.hikcreate.utils.CommonUtils
import org.apache.spark.sql.SparkSession
class Train extends OfflineBase with Logging {
override protected def doTask(sparkSession: SparkSession): Unit = {
sparkSession.sql("set hive.exec.dynamic.partition=true")
sparkSession.sql("set hive.exec.dynamic.partition.mode=nostrict")
val DWB_safe_drive_sql ="""select * from
|table""".stripMargin
val safeTotal = sparkSession.sql(DWB_safe_drive_sql)
CommonUtils.insertDateIntoTable(safeTotal, "overwrite", "dwb_topic.dwb_safe_total")
}
}
src/main/scala/com/hikcreate/utils/CommonUtils.scala 0 → 100644
package com.hikcreate.utils
import java.io._
import java.net.{HttpURLConnection, URL}
import java.text.SimpleDateFormat
import java.util
import java.util._
import com.alibaba.fastjson.{JSON, JSONArray, JSONObject}
import com.hikcreate.config.SparkConfig
import org.apache.spark.sql.{DataFrame, SparkSession}
import scala.collection.JavaConversions._
import scalaj.http.Http
/**
* 通用工具类
*/
object CommonUtils {
/**
* 获取sparkSession
*
* @return
*/
def getSparkSession(): SparkSession = {
val session = SparkSession
.builder()
// .master(SparkConfig.getMaster())
// .appName(SparkConfig.getAppName())
.enableHiveSupport()
.config("hive.exec.dynamic.partition", true) // 支持 Hive 动态分区
.config("hive.exec.dynamic.partition.mode", "nonstrict") // 非严格模式
.config("hive.metastore.uris", "thrift://hadoop01:9083")
.config("spark.sql.sources.partitionOverwriteMode", "dynamic")
.config("spark.sql.crossJoin.enabled", "true")
.getOrCreate()
session
}
/**
* 获取指定配置文件中的属性
*
* @return
*/
def getProperties(filename: String): Properties = {
val properties = new Properties()
properties.load(this.getClass.getClassLoader.getResourceAsStream(filename))
/*// 指定配置文件路径
// val path = Thread.currentThread().getContextClassLoader.getResource(filename).getPath
val path = "/home/taskConf/" + filename
// 加载配置文件内容
properties.load(new FileInputStream(path))*/
// 返回
properties
}
/**
* 创建UUID
*
* @return
*/
def createUUID(): String = {
UUID.randomUUID().toString.replace("-", "").toUpperCase
}
/**
* 保存数据到hive
*
* @param dataFrame
* @param mode
* @param tableName
*/
def saveDataToHive(dataFrame: DataFrame, mode: String, partitionday: String, tableName: String): Unit = {
if (null != partitionday) {
dataFrame.repartition(1)
.write
.mode(mode)
.format("hive")
.partitionBy(partitionday)
.saveAsTable(tableName)
} else {
dataFrame
.write
.mode(mode)
.format("hive")
.saveAsTable(tableName)
}
}
def insertDateIntoTable(dataFrame: DataFrame, mode: String, tableName: String): Unit = {
dataFrame.repartition(1)
.write
.mode(mode)
.format("hive")
.insertInto(tableName)
}
//计算是否2个日期在30天内
def yearToYearTheDays(year2: Date): Boolean = {
//val format2 = new SimpleDateFormat("yyyy-MM-dd")
val betweenTimes = new Date().getTime - year2.getTime
val days = (betweenTimes / 1000 / 3600 / 24) - 1
if (days >= 0 && days <= 30)
true
else
false
}
/**
* 从ods层写数据到dwd层
*
* @param ods_table
* @param dwd_table
* @param partitions
* @return
*/
def odstodwdCommon(ods_table: String, dwd_table: String, partitions: String, partitionday: String): String = {
val sql =
s"""
|INSERT INTO TABLE ${dwd_table} SELECT * from ${ods_table} where ${partitionday}='${partitions}'
""".stripMargin
return sql
}
/**
* 获取指定日期前几天的年月日
*
* @param dateString 时间字符串
* @param beforeDays 几天前
* @return
*/
def getBeforeDate(dateString: String, beforeDays: Int, format: String): String = {
// 将时间字符串转换为时间类型
val date = new SimpleDateFormat(getStrFormat(dateString)).parse(dateString)
// 获取指定时间是一年中的第一天
val calendar = Calendar.getInstance()
calendar.setTime(date)
val dayOfYear = calendar.get(Calendar.DAY_OF_YEAR)
// 获取指定日期几天前的日期
calendar.set(Calendar.DAY_OF_YEAR, dayOfYear - beforeDays)
val preDay = new SimpleDateFormat(format).format(calendar.getTime)
preDay
}
/**
* 获取输入的时间字符串的格式
*
* @param dateString 时间字符串
* @return 时间字符串对应的格式
*/
private def getStrFormat(dateString: String): String = {
// 要返回的时间格式
var format = ""
// 判断时间字符串格式
if (dateString.contains("-") && dateString.length == 10) {
format = "yyyy-MM-dd"
} else if (dateString.contains("/") && dateString.length == 10) {
format = "yyyy/MM/dd"
} else if (dateString.length == 8) {
format = "yyyyMMdd"
}
format
}
/**
* 角度换算成弧度
*
* @param d 角度
* @return
*/
def rad(d: Double): Double = {
return d * Math.PI / 180.0
}
/**
* 根据经纬度计算两点间的球面距离
*
* @param lat1 纬度1
* @param lng1 经度1
* @param lat2 纬度2
* @param lng2 经度2
* @return
*/
def getDistance(lat1: Double, lng1: Double, lat2: Double, lng2: Double): Double = {
var s = 0D
if (null != lat1 && null != lng1 && null != lat2 && null != lng2) {
val EARTH_RADIUS = 6371.830D
val radLat1 = rad(lat1)
val radLat2 = rad(lat2)
val radLng1 = rad(lng1)
val radLng2 = rad(lng2)
//var s = 2 * Math.asin(Math.sqrt(Math.pow(Math.sin(a / 2), 2) + Math.cos(radLat1) * Math.cos(radLat2) * Math.pow(Math.sin(b / 2), 2)))
s = Math.acos(Math.cos(radLat1) * Math.cos(radLat2) * Math.cos(radLng1 - radLng2) + Math.sin(radLat1) * Math.sin(radLat2))
s = s * EARTH_RADIUS
s = Math.round(s * 1000D) / 1000D
s
} else {
0D
}
}
/**
* 计算里程数 车辆行驶里程和行驶时间
*
* @param len_list
* @return
*/
def getAddressLen(len_list: List[String]): List[Seq[String]] = {
var leg = 0D
var arr: util.ArrayList[Long] = new util.ArrayList[Long]()
val daytime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
var li = new util.ArrayList[String]()
for (x <- 0 to len_list.length - 2) {
val lonanlat = len_list.get(x).split(";")
val lonandlat_next = len_list.get(x + 1).split(";")
if ("null" != lonanlat(3) && "null" != lonandlat_next(1)) {
if ("null" != lonanlat(1)) {
li.add(len_list.get(x))
} else {
li.add(lonanlat(0) + ";" + lonandlat_next(1) + ";" + lonandlat_next(2) + ";" + lonanlat(3) + ";" + lonanlat(4))
}
} else if ("null" != lonanlat(1) && "null" != lonandlat_next(3)) {
if ("null" != lonanlat(3)) {
li.add(len_list.get(x))
} else {
li.add(lonandlat_next(0) + ";" + lonanlat(1) + ";" + lonanlat(2) + ";" + lonandlat_next(3) + ";" + lonandlat_next(4))
}
}
}
getDriverLength(li)
}
/**
* 计算行驶里程
*
* @param arr
* @return
*/
def getDriverLength(arr: util.ArrayList[String]): List[Seq[String]] = {
val arrs = arr.sortBy(x => x.substring(0, 19))
var leg = 0D
var arlis = new util.ArrayList[Long]()
var arr_li = new util.ArrayList[Tuple4[String, String, Double, Double]]()
val daytime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
for (x <- 0 to arrs.length - 2) {
val str = arrs(x).split(";")
val str1 = arrs(x + 1).split(",")
val time1 = daytime.parse(str(0)).getTime
val time2 = daytime.parse(str1(0)).getTime
val len = getDistance(str(4).toDouble, str(3).toDouble, str1(4).toDouble, str1(3).toDouble)
if (len != 0 && time1 != time2 && str(2) == str1(2)) {
val times = time2 - time1
arr_li.add((str(1), str(2), times, len))
}
}
val tupleToTuple = arr_li.groupBy(x => (x._1, x._2)).mapValues(x => (x.map(x => x._3).reduce(_ + _), x.map(x => x._4).reduce(_ + _)))
val tuples = tupleToTuple.map(x => Seq(x._1._1, x._1._2, x._2._1.toString, x._2._2.toString))
tuples.toList
}
/**
* 根据经纬度获取地理位置
*
* @param lon
* @param lat
* @return
*/
def interfaceUrl(lon: String, lat: String): String = {
try {
val path = "http://10.197.236.100:40612/bcpbase/geocode/regeo"
val url = new URL(path)
val conn: HttpURLConnection = url.openConnection().asInstanceOf[HttpURLConnection]
var out: PrintWriter = null
//设置通用的请求属性
conn.setRequestProperty("accept", "*/*")
// 连接复用
conn.setRequestProperty("connection", "Keep-Alive")
conn.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)")
//设置是否向httpUrlConnection输出，设置是否从httpUrlConnection读入，此外发送post请求必须设置这两个
//最常用的Http请求无非是get和post，get请求可以获取静态页面，也可以把参数放在URL字串后面，传递给servlet，
//post与get的 不同之处在于post的参数不是放在URL字串里面，而是放在http请求的正文内。
conn.setDoOutput(true)
conn.setDoInput(true)
//获取URLConnection对象对应的输出流
out = new PrintWriter(conn.getOutputStream())
//发送请求参数即数据
out.print(lat)
//缓冲数据
out.flush()
//获取URLConnection对象对应的输入流
val is = conn.getInputStream()
//构造一个字符流缓存
val br = new BufferedReader(new InputStreamReader(is, "GBK"))
var str = ""
var flage = true
while (flage) {
val s = br.readLine()
if (s == null) {
flage = false
} else {
str += s
}
}
//关闭流
is.close()
conn.disconnect()
str
} catch {
case e: Exception => e.printStackTrace()
null
}
}
import java.text.SimpleDateFormat
/**
* 两个时间差
*
* @param a
* @param b
* @return 秒数
*/
def getBetweenSecond(a: String, b: String): Long = {
val format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
var startDate: Date = null
var endDate: Date = null
var ss = 0L
startDate = format.parse(a)
endDate = format.parse(b)
try {
ss = endDate.getTime - startDate.getTime
} catch {
case e: Exception =>
e.printStackTrace()
}
ss / 1000
}
def getAddressAndLocationCode(lon: Double, lat: Double): (String, String) = {
val json = new JSONObject()
val arr = new JSONArray()
val lonAndLat = new JSONObject()
lonAndLat.put("longitude", lon)
lonAndLat.put("latitude", lat)
arr.add(lonAndLat)
json.put("locations", arr)
val response = Http("http://10.197.236.100:40612/bcpbase/geocode/regeo")
.postData(json.toJSONString)
.header("content-type", "application/json")
//.charset("ISO-8859-1")
.timeout(connTimeoutMs = 8000, readTimeoutMs = 8000)
.asString
if (response.code == 200) {
val body = JSON.parseObject(response.body)
val item = body.getJSONObject("result").getJSONArray("regeoItems").getJSONObject(0)
val address = item.getString("formattedAddress")
val locationCode = item.getJSONObject("addressComponent").getString("adcode")
(address, locationCode)
} else {
throw new RuntimeException("http请求城市区编码出错")
}
}
//根据经纬度获取地区编码
def getLocationCode(lon: Double, lat: Double): (String, String, String) = {
val locationCode = getAddressAndLocationCode(lon, lat)._2
val provinceCode = locationCode.substring(0, 2)
val cityCode = locationCode.substring(2, 4)
val areaCode = locationCode.substring(4, 6)
(provinceCode, cityCode, areaCode)
}
def isIDNumber(IDNumber: String): Boolean = {
if (IDNumber == null || "" == IDNumber) return false
// 定义判别用户身份证号的正则表达式（15位或者18位，最后一位可以为字母）
val regularExpression = "(^[1-9]\\d{5}(18|19|20)\\d{2}((0[1-9])|(10|11|12))(([0-2][1-9])|10|20|30|31)\\d{3}[0-9Xx]$)|" + "(^[1-9]\\d{5}\\d{2}((0[1-9])|(10|11|12))(([0-2][1-9])|10|20|30|31)\\d{3}$)"
val matches = IDNumber.matches(regularExpression)
//判断第18位校验值
if (matches) if (IDNumber.length == 18) try {
val charArray = IDNumber.toCharArray
//前十七位加权因子
val idCardWi = Array(7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2)
//这是除以11后，可能产生的11位余数对应的验证码
val idCardY = Array("1", "0", "X", "9", "8", "7", "6", "5", "4", "3", "2")
var sum = 0
var i = 0
while ( {
i < idCardWi.length
}) {
val current = String.valueOf(charArray(i)).toInt
val count = current * idCardWi(i)
sum += count
{
i += 1;
i - 1
}
}
val idCardLast = charArray(17)
val idCardMod = sum % 11
if (idCardY(idCardMod).toUpperCase == String.valueOf(idCardLast).toUpperCase) return true
else {
return false
}
} catch {
case e: Exception =>
e.printStackTrace()
return false
}
matches
}
}
src/main/scala/com/hikcreate/utils/DateUtils.scala 0 → 100644
package com.hikcreate.utils
import java.util.Date
import org.apache.commons.lang3.time.DateFormatUtils
object DateUtils {
private val parsePatterns = Array("yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy-MM", "yyyy/MM/dd", "yyyy/MM/dd HH:mm:ss",
"yyyy/MM/dd HH:mm", "yyyy/MM", "yyyy.MM.dd", "yyyy.MM.dd HH:mm:ss", "yyyy.MM.dd HH:mm", "yyyy.MM")
def getDate: String = DateFormatUtils.format(new Date, "yyyy-MM-dd");
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment