Skip to content

D
datatoHbaseAdEs
Commit 0134edf0 by wangkai
Options

斑马信用演示环境数据入库代码

parents
Showing with 943 additions and 0 deletions
hkys_gy/.idea/compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="hkys_gy" />
</profile>
</annotationProcessing>
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/hydra.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="HydraSettings">
<option name="hydraStorePath" value="G:\sparkCore\hkys_gy\.hydra\idea" />
<option name="noOfCores" value="2" />
<option name="projectRoot" value="G:\sparkCore\hkys_gy" />
<option name="sourcePartitioner" value="auto" />
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/misc.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/scala_compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaCompilerConfiguration">
<profile name="Maven 1" modules="hkys_gy">
<parameters>
<parameter value="-target:jvm-1.7" />
</parameters>
</profile>
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/uiDesigner.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>
\ No newline at end of file
hkys_gy/hkys_gy.iml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />
\ No newline at end of file
hkys_gy/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.hikcreate</groupId>
<artifactId>hkys_gy</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<hbase.version>1.1.0-cdh5.15.1</hbase.version>
<spark.version>2.3.0</spark.version>
<scala.version>2.11</scala.version>
<fastjson.version>1.2.38</fastjson.version>
<mysqlDriver.version>5.1.47</mysqlDriver.version>
<odbc.version>12.1.0.1-atlassian-hosted</odbc.version>
<impala.driver.version>2.6.4</impala.driver.version>
<kudu.version>1.10.0</kudu.version>
<netty.version>4.4.42</netty.version>
<kafka.version>2.1.1</kafka.version>
<!-- <kafka.version>0.10.0.1</kafka.version>-->
<!--<hbase.version>1.2.5</hbase.version>-->
<hadoop.version>2.7.3</hadoop.version>
<slf4j.version>1.7.1</slf4j.version>
<elsticsearch.version>6.5.4</elsticsearch.version>
</properties>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<!-- 引入spark相关的jar包 -->
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
<artifactId>joda-time</artifactId>
<groupId>joda-time</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<!-- 引入fastjson -->
<!--<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${fastjson.version}</version>
</dependency>-->
<!-- 引入es与spark的集成 -->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-spark-20_2.11</artifactId>
<version>6.5.4</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.5.4</version>
</dependency>
<!--引入hbase -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.1.1</version>
</dependency>
<!-- <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.0.42.Final</version>
</dependency>-->
<dependency>
<groupId>fakepath</groupId>
<artifactId>hbase-spark</artifactId>
<version>1.0.0</version>
<scope>system</scope>
<systemPath>D:/maven/apache-maven-3.5.0/lib/hbase-spark-1.0.0.jar</systemPath>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<args>
<arg>-target:jvm-1.7</arg>
</args>
<jvmArgs>
<jvmArg>-Xss6000K</jvmArg>
</jvmArgs>
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.18.1</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala 0 → 100644
package com.hikcreate.DataToHbase
import com.hikcreate.Utils.{DataToHbaseUtils, DateFormatUtils}
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapreduce._
import org.apache.spark.sql.SparkSession
import org.apache.spark.{SparkConf, SparkContext}
object WriteToHbase extends Serializable {
def main(args: Array[String]): Unit = {
//告知哪些类型需要序列化
val conf = new SparkConf()
.setAppName("toHbase")
.setMaster("local[*]")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.registerKryoClasses(Array(classOf[ImmutableBytesWritable],classOf[DateFormatUtils]))
val sc = new SparkContext(conf)
val spark = SparkSession.builder().config(conf).getOrCreate()
if (args.length < 0) {
println("传递的参数错误")
System.exit(1)
}
//hbase表名
val tablename=args(0)
val job:Job =new DataToHbaseUtils().DataToHbaseConf(tablename)
val dateUtils=new DateFormatUtils
import spark.implicits._
val datacsv = spark.read.format("com.databricks.spark.csv")
.option("header", "true") //在csv第一行有属性"true"，没有就是"false"
//.load("C:\\Users\\Administrator\\Documents\\drivinglicense.csv")
.load("/root/drivinglicense.csv")
//datacsv.createOrReplaceTempView("drivinglicense")
/*val datafmt=spark.sql(
"""
|select
|DABH,
|SFZMHM,
|ZJCX,
|YZJCX,
|cast(date_format(to_timestamp(QFRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as QFRQ,
|cast(date_format(to_timestamp(SYRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as SYRQ,
|cast(date_format(to_timestamp(CCLZRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as CCLZRQ,
|CCFZJG,
|JZQX,
|cast(date_format(to_timestamp(YXQS, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as YXQS,
|cast(date_format(to_timestamp(YXQZ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as YXQZ,
|LJJF,
|CFRQ,
|XXTZRQ,
|BZCS,
|ZT,
|LY,
|JXMC,
|JLY,
|XZQH,
|XZQJ,
|cast(date_format(to_timestamp(FZRQ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as FZRQ,
|JBR,
|GLBM,
|FZJG,
|cast(date_format(to_timestamp(GXSJ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as GXSJ,
|LSH,
|XGZL,
|BZ,
|JYW,
|YDABH,
|SQDM,
|ZXBH,
|XH,
|SFZMMC,
|HMCD,
|XM,
|XB,
|cast(date_format(to_timestamp(CSRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as CSRQ,
|GJ,
|DJZSXZQH,
|DJZSXXDZ,
|LXZSXZQH,
|LXZSXXDZ,
|LXZSYZBM,
|LXDH,
|SJHM,
|DZYX,
|ZZZM,
|ZZFZJG,
|ZZFZRQ,
|SFBD,
|DWBH,
|cast(date_format(to_timestamp(SYYXQZ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as SYYXQZ,
|XCZG,
|XCZJCX,
|RYZT
|from drivinglicense
""".stripMargin)*/
//datafmt.show()
val datacsvDF = datacsv.map(r => {
Drivinglicense(
r.getAs[String]("DABH"),
r.getAs[String]("SFZMHM"),
r.getAs[String]("ZJCX"),
r.getAs[String]("YZJCX"),
dateUtils.dateFormatRes(r.getAs[String]("QFRQ")),
dateUtils.dateFormatRes(r.getAs[String]("SYRQ")),
dateUtils.dateFormatRes(r.getAs[String]("CCLZRQ")),
r.getAs[String]("CCFZJG"),
r.getAs[String]("JZQX"),
dateUtils.dateFormatRes(r.getAs[String]("YXQS")),
dateUtils.dateFormatRes(r.getAs[String]("YXQZ")),
r.getAs[String]("LJJF"),
dateUtils.dateFormatRes(r.getAs[String]("CFRQ")),
dateUtils.dateFormatRes(r.getAs[String]("XXTZRQ")),
r.getAs[String]("BZCS"),
r.getAs[String]("ZT"),
r.getAs[String]("LY"),
r.getAs[String]("JXMC"),
r.getAs[String]("JLY"),
r.getAs[String]("XZQH"),
r.getAs[String]("XZQJ"),
dateUtils.dateFormatRes(r.getAs[String]("FZRQ")),
r.getAs[String]("JBR"),
r.getAs[String]("GLBM"),
r.getAs[String]("FZJG"),
dateUtils.dateFormatRes(r.getAs[String]("GXSJ")),
r.getAs[String]("LSH"),
r.getAs[String]("XGZL"),
r.getAs[String]("BZ"),
r.getAs[String]("JYW"),
r.getAs[String]("YDABH"),
r.getAs[String]("SQDM"),
r.getAs[String]("ZXBH"),
r.getAs[String]("XH"),
r.getAs[String]("SFZMMC"),
r.getAs[String]("HMCD"),
r.getAs[String]("XM"),
r.getAs[String]("XB"),
dateUtils.dateFormatRes(r.getAs[String]("CSRQ")),
r.getAs[String]("GJ"),
r.getAs[String]("DJZSXZQH"),
r.getAs[String]("DJZSXXDZ"),
r.getAs[String]("LXZSXZQH"),
r.getAs[String]("LXZSXXDZ"),
r.getAs[String]("LXZSYZBM"),
r.getAs[String]("LXDH"),
r.getAs[String]("SJHM"),
r.getAs[String]("DZYX"),
r.getAs[String]("ZZZM"),
r.getAs[String]("ZZFZJG"),
r.getAs[String]("ZZFZRQ"),
r.getAs[String]("SFBD"),
r.getAs[String]("DWBH"),
dateUtils.dateFormatRes(r.getAs[String]("SYYXQZ")),
r.getAs[String]("XCZG"),
r.getAs[String]("XCZJCX"),
r.getAs[String]("RYZT")
)
}).toDF()
datacsvDF.show()
var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => {
val rowkey = Bytes.toBytes(row.getAs[String]("SFZMHM"))
columns.map(
column => {
val colvalue = if(row.getAs[String](column) ==null) "null" else row.getAs[String](column)
(rowkey, column, colvalue)
}).toList
})
val rerdd=kvRDD.rdd.map(value => {
val family=Bytes.toBytes("info")
val key=value._1
val col=value._2
val put = new Put(value._1)
if(!col.isEmpty) {
put.addColumn(family, Bytes.toBytes(col),Bytes.toBytes(value._3))
}
(new ImmutableBytesWritable, put)
})
rerdd.take(5).foreach(x=>{println(x)})
rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
sc.stop()
}
case class Drivinglicense(DABH: String,
SFZMHM: String,
ZJCX: String,
YZJCX: String,
QFRQ: String,
SYRQ: String,
CCLZRQ: String,
CCFZJG: String,
JZQX: String,
YXQS: String,
YXQZ: String,
LJJF: String,
CFRQ: String,
XXTZRQ: String,
BZCS: String,
ZT: String,
LY: String,
JXMC: String,
JLY: String,
XZQH: String,
XZQJ: String,
FZRQ: String,
JBR: String,
GLBM: String,
FZJG: String,
GXSJ: String,
LSH: String,
XGZL: String,
BZ: String,
JYW: String,
YDABH: String,
SQDM: String,
ZXBH: String,
XH: String,
SFZMMC: String,
HMCD: String,
XM: String,
XB: String,
CSRQ: String,
GJ: String,
DJZSXZQH: String,
DJZSXXDZ: String,
LXZSXZQH: String,
LXZSXXDZ: String,
LXZSYZBM: String,
LXDH: String,
SJHM: String,
DZYX: String,
ZZZM: String,
ZZFZJG: String,
ZZFZRQ: String,
SFBD: String,
DWBH: String,
SYYXQZ: String,
XCZG: String,
XCZJCX: String,
RYZT: String
)
}
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala 0 → 100644
package com.hikcreate.Utils
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
import org.apache.hadoop.mapreduce.Job
class DataToHbaseUtils {
def DataToHbaseConf(tablename:String) = {
val hbaseconf = HBaseConfiguration.create()
hbaseconf.set("hbase.zookeeper.quorum", "10.197.236.216,10.197.236.217,10.197.236.218")
hbaseconf.set("hbase.zookeeper.property.clientPort", "2181")
hbaseconf.set("hbase.defaults.for.version.skip", "true")
hbaseconf.set("zookeeper.znode.parent", "/hbase-unsecure")
hbaseconf.set(TableOutputFormat.OUTPUT_TABLE, tablename)
//初始化jobconf，TableOutputFormat必须是org.apache.hadoop.hbase.mapred包下的
val job = Job.getInstance(hbaseconf)
job.setOutputFormatClass(classOf[TableOutputFormat[ImmutableBytesWritable]])
job.setOutputValueClass(classOf[Put])
job.setOutputKeyClass(classOf[ImmutableBytesWritable])
job
}
}
hkys_gy/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala 0 → 100644
package com.hikcreate.Utils
class DateFormatUtils extends Serializable {
def dateFormatRes(datetime: String) = {
if (datetime == null) {
"null"
} else {
val datemk = new DateUtils(datetime)
val year = datemk.YEAR
val month = if (datemk.MONTH < 10)s"""0${datemk.MONTH}""" else datemk.MONTH
val day = if (datemk.DAY_OF_MONTH < 10)s"""0${datemk.DAY_OF_MONTH}""" else datemk.DAY_OF_MONTH
val hour = if (datemk.HOUR_OF_DAY < 10)s"""0${datemk.HOUR_OF_DAY}""" else datemk.HOUR_OF_DAY
val minute = if (datemk.MINUTE < 10)s"""0${datemk.MINUTE}""" else datemk.MINUTE
val second = if (datemk.SECOND < 10)s"""0${datemk.SECOND}""" else datemk.SECOND
(year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second)
}
}
}
hkys_gy/src/main/scala/com/hikcreate/Utils/DateUtils.scala 0 → 100644
package com.hikcreate.Utils
import java.text.SimpleDateFormat
import java.util.{Calendar, Date}
class DateUtils(datetime: String){
val date: Date =toDate()
private val calendar: Calendar = getCalendar()
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val YEAR: Int = if(calendar == null) -1 else calendar.get(Calendar.YEAR)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val MONTH: Int = if(calendar == null) -1 else calendar.get(Calendar.MONTH) + 1
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val QUARTER: Int = if(calendar == null) -1 else getQuarter()
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val WEEK_OF_YEAR: Int = if(calendar == null) -1 else calendar.get(Calendar.WEEK_OF_YEAR)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val WEEK_OF_MONTH: Int = if(calendar == null) -1 else calendar.get(Calendar.WEEK_OF_MONTH)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val DAY_OF_YEAR: Int = if(calendar == null) -1 else calendar.get(Calendar.DAY_OF_YEAR)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val DAY_OF_MONTH: Int = if(calendar == null) -1 else calendar.get(Calendar.DAY_OF_MONTH)
/**
* 若datetime为"null"，则值为-1，如果day_of_week为1，则值为7，否则为day_of_week-1
*/
val DAY_OF_WEEK: Int = if(calendar == null) -1 else if(calendar.get(Calendar.DAY_OF_WEEK) == 1) 7 else calendar.get(Calendar.DAY_OF_WEEK) -1
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val HOUR_OF_DAY: Int = if(calendar == null) -1 else calendar.get(Calendar.HOUR_OF_DAY)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val MINUTE: Int = if(calendar == null) -1 else calendar.get(Calendar.MINUTE)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val SECOND: Int = if(calendar == null) -1 else calendar.get(Calendar.SECOND)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val MILLISECOND: Int = if(calendar == null) -1 else calendar.get(Calendar.MILLISECOND)
/**
* 若datetime为"null"，则值为null，否则为其正常值
*/
val FIRST_DAY_OF_MONTH: String = if(calendar == null) null else s"""$YEAR-${if(MONTH < 10) "0" + MONTH else MONTH}-01"""
/**
* 若datetime为"null"，则值为null，否则为其正常值
*/
val FIRST_DAY_OF_YEAR: String = if(calendar == null) null else s"""$YEAR-01-01"""
def toString(format: String): String = {
if(date == null){
null
} else {
val quarter = "0" + QUARTER
val month = if(MONTH < 10) s"""0$MONTH""" else MONTH
val dayOfMonth = if(DAY_OF_MONTH < 10) s"""0$DAY_OF_MONTH""" else DAY_OF_MONTH
val hourOfDay = if(HOUR_OF_DAY < 10) s"""0$HOUR_OF_DAY""" else HOUR_OF_DAY
val minute = if(MINUTE < 10) s"""0$MINUTE""" else MINUTE
val second = if(SECOND < 10) s"""0$SECOND""" else SECOND
val weekOfYear = if(WEEK_OF_YEAR < 10) s"""0$WEEK_OF_YEAR""" else WEEK_OF_YEAR
val weekOfMonth = if(WEEK_OF_MONTH < 10) s"""0$WEEK_OF_MONTH""" else WEEK_OF_MONTH
val formatList = List(
"yyyy",
"QQ",
"MM",
"M",
"dd",
"d",
"HH",
"mm",
"ss",
"yyyyMM",
"yyyy/MM",
"yyyy-MM",
"yyyy年MM月",
"yyyyMMdd",
"yyyy/MM/dd",
"yyyy-MM-dd",
"yyyy年MM月dd日",
"MMdd",
"MM/dd",
"MM-dd",
"MM月dd日",
"yyyyMMdd HH:mm:ss",
"yyyy/MM/dd HH:mm:ss",
"yyyy-MM-dd HH:mm:ss",
"yyyy年MM月dd日 HH时mm分ss秒",
"yyyy年第ww周",
"yyyy年MM月第WW周",
"yyyy年第QQ季度",
"yyyy年QQ季度"
)
if (formatList.contains(format)){
format
.replace("yyyy", s"""$YEAR""")
.replace("MM", s"""$month""")
.replace("M", s"""$MONTH""")
.replace("dd", s"""$dayOfMonth""")
.replace("d", s"""$DAY_OF_MONTH""")
.replace("HH", s"""$hourOfDay""")
.replace("mm", s"""$minute""")
.replace("ss", s"""$second""")
.replace("ww", s"""$weekOfYear""")
.replace("WW", s"""$weekOfMonth""")
.replace("QQ", s"""$quarter""")
} else {
throw new RuntimeException(
s"""
|时间格式错误，当前只支持以下23种格式：
|
|yyyy,
|QQ,
|MM,
|dd,
|HH,
|mm,
|ss,
|yyyyMM,
|yyyy/MM,
|yyyy-MM,
|yyyy年MM月,
|yyyyMMdd,
|yyyy/MM/dd,
|yyyy-MM-dd,
|yyyy年MM月dd日,
|MMdd,
|MM/dd,
|MM-dd,
|MM月dd日,
|yyyyMMdd HH:mm:ss,
|yyyy/MM/dd HH:mm:ss,
|yyyy-MM-dd HH:mm:ss,
|yyyy年MM月dd日 HH时mm分ss秒,
|yyyy年第ww周,
|yyyy年MM月第WW周,
|yyyy年第QQ季度,
|yyyy年QQ季度
|
|您输入的时间格式字符串为：$format
|""".stripMargin)
}
}
}
/**
* 作者：王志豪
* 方法功能：获取calendar对象
* 处理逻辑：判断date是否为空，如果为空则返回null，否则返回date对应的calendar
* @return date对应的calendar
*/
private def getCalendar(): Calendar = {
if(date == null){
null
} else {
val cal = Calendar.getInstance()
cal.setFirstDayOfWeek(Calendar.MONDAY)
cal.setTime(date)
cal
}
}
/**
* 作者：王志豪
* 方法功能：获取date对应的季度
* 处理逻辑：1月到3月为1季度，3月到6月为2季度，7月到9月为3季度，10月到12月为4季度。不在此范围则抛出异常
* @return 返回date对应的季度
*/
private def getQuarter(): Int = {
MONTH match {
case 1 => 1
case 2 => 1
case 3 => 1
case 4 => 2
case 5 => 2
case 6 => 2
case 7 => 3
case 8 => 3
case 9 => 3
case 10 => 4
case 11 => 4
case 12 => 4
case _ => throw new RuntimeException("月份超出正常值外")
}
}
private def toDate(): Date = {
if(datetime == null){
new Date()
} else if ("null".equals(datetime)) {
null
} else {
new SimpleDateFormat(parseFormat(datetime)).parse(datetime)
}
}
/**
* 作者：王志豪
* 方法功能：解析时间格式
* 处理逻辑：1、判断字符串是否为空，如果为空则返回null，不为空则舍去毫秒
* 2、正则匹配时间字符串的格式，并返回格式。
* @param dateString 时间字符串
* @return 时间字符串datestring所对应的时间格式。
*/
private def parseFormat(dateString: String): String = {
val time = if(dateString != null) dateString.split("\\.")(0) else null
if(time == null){
null
} else if(time.matches("^\\d{6}$")){
"yyyyMM"
} else if(time.matches("^\\d{4}/\\d{2}$")){
"yyyy/MM"
} else if(time.matches("^\\d{4}-\\d{2}$")){
"yyyy-MM"
} else if(time.matches("^\\d{4}年\\d{2}月$")){
"yyyy年MM月"
} else if(time.matches("^\\d{8}$")){
"yyyyMMdd"
} else if(time.matches("^\\d{4}/\\d{2}/\\d{2}$")){
"yyyy/MM/dd"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{1}$")){
"yyyy/MM/dd"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{2}$")){
"yyyy/MM/dd"
} else if(time.matches("^\\d{4}/\\d{2}/\\d{1}$")){
"yyyy/MM/dd"
} else if(time.matches("^\\d{4}-\\d{2}-\\d{2}$")){
"yyyy-MM-dd"
} else if(time.matches("^\\d{4}年\\d{2}月\\d{2}日$")){
"yyyy年MM月dd日"
} else if(time.matches("^\\d{8}\\s\\d{2}:\\d{2}:\\d{2}$")){
"yyyyMMdd HH:mm:ss"
} else if(time.matches("^\\d{4}/\\d{2}/\\d{2}\\s\\d{2}:\\d{2}:\\d{2}$")){
"yyyy/MM/dd HH:mm:ss"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{2}\\s\\d{2}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{1}\\s\\d{2}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{1}\\s\\d{1}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
}else if(time.matches("^\\d{4}/\\d{2}/\\d{2}\\s\\d{1}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
}else if(time.matches("^\\d{4}/\\d{1}/\\d{2}\\s\\d{1}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
}else if(time.matches("^\\d{4}/\\d{2}/\\d{1}\\s\\d{2}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
}else if(time.matches("^\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}$")){
"yyyy-MM-dd HH:mm:ss"
} else {
throw new RuntimeException(
s"""
|时间格式错误，当前只支持以下11种格式：
|yyyyMM，
|yyyy-MM，
|yyyy/MM，
|yyyy年MM月，
|yyyyMMdd，
|yyyy-MM-dd，
|yyyy/MM/dd，
|yyyy年MM月dd日，
|yyyyMMdd HH:mm:ss，
|yyyy-MM-dd HH:mm:ss，
|yyyy/MM/dd HH:mm:ss。
|需要解析的时间字符串为：$dateString
|""".stripMargin)
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment