Skip to content

D
datatoHbaseAdEs
Commit fac63730 by wangkai
Options

字段类型转换和schema大小写转换

parent 0134edf0
Showing with 790 additions and 289 deletions
hkys_gy/.idea/workspace.xml
...@@ -30,26 +30,19 @@ ...@@ -30,26 +30,19 @@
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false"> <file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml"> <entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="152"> <state relative-caret-position="3151">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" /> <caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file leaf-file-name="hkys_gy.iml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/hkys_gy.iml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file leaf-file-name="WriteToHbase.scala" pinned="false" current-in-tab="false"> <file leaf-file-name="WriteToHbase.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="179"> <state relative-caret-position="3933">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" /> <caret line="177" column="18" selection-start-line="177" selection-start-column="18" selection-end-line="177" selection-end-column="18" />
<folding> <folding>
<element signature="e#35#97#0" expanded="true" /> <element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
...@@ -58,7 +51,7 @@ ...@@ -58,7 +51,7 @@
<file leaf-file-name="DateUtils.scala" pinned="false" current-in-tab="false"> <file leaf-file-name="DateUtils.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92"> <state relative-caret-position="69">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" /> <caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding> <folding>
<element signature="e#29#62#0" expanded="true" /> <element signature="e#29#62#0" expanded="true" />
...@@ -67,15 +60,15 @@ ...@@ -67,15 +60,15 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file leaf-file-name="VehicleToHbase.scala" pinned="false" current-in-tab="false"> <file leaf-file-name="VehicleToHbase.scala" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188"> <state relative-caret-position="1223">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" /> <caret line="340" column="8" selection-start-line="340" selection-start-column="8" selection-end-line="340" selection-end-column="8" />
<folding> <folding>
<element signature="e#35#97#0" expanded="true" /> <element signature="e#35#97#0" expanded="true" />
<element signature="e#2814#2864#0" /> <element signature="e#1856#5751#0" />
<element signature="e#6726#11186#0" /> <element signature="e#5786#10621#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
...@@ -84,18 +77,47 @@ ...@@ -84,18 +77,47 @@
<file leaf-file-name="VioViolationToHbase.scala" pinned="false" current-in-tab="false"> <file leaf-file-name="VioViolationToHbase.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188"> <state relative-caret-position="-5368">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" /> <caret line="59" column="6" lean-forward="true" selection-start-line="59" selection-start-column="6" selection-end-line="59" selection-end-column="6" />
<folding> <folding>
<element signature="e#131#193#0" expanded="true" /> <element signature="e#131#161#0" expanded="true" />
<element signature="e#2032#4181#0" /> <element signature="e#1977#4126#0" />
<element signature="e#4922#8349#0" /> <element signature="e#4201#7753#0" />
<element signature="e#7797#11367#0" />
<element signature="e#11463#13112#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file leaf-file-name="DateFormatUtils.scala" pinned="false" current-in-tab="true"> <file leaf-file-name="Test.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/Test.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="667">
<caret line="29" column="4" selection-start-line="29" selection-start-column="4" selection-end-line="29" selection-end-column="4" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="StringLike.class" pinned="false" current-in-tab="false">
<entry file="jar://D:/scala-2.11.8/scala-2.11.8/lib/scala-library.jar!/scala/collection/immutable/StringLike.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="277">
<caret line="32" column="6" selection-start-line="32" selection-start-column="6" selection-end-line="32" selection-end-column="6" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="MapElementsExec.class" pinned="false" current-in-tab="false">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-sql_2.11/2.3.0/spark-sql_2.11-2.3.0.jar!/org/apache/spark/sql/execution/MapElementsExec.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
<caret line="9" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DateFormatUtils.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46"> <state relative-caret-position="46">
...@@ -107,7 +129,7 @@ ...@@ -107,7 +129,7 @@
<file leaf-file-name="DataToHbaseUtils.scala" pinned="false" current-in-tab="false"> <file leaf-file-name="DataToHbaseUtils.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115"> <state relative-caret-position="92">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" /> <caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding> <folding>
<element signature="e#29#78#0" expanded="true" /> <element signature="e#29#78#0" expanded="true" />
...@@ -138,6 +160,10 @@ ...@@ -138,6 +160,10 @@
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/WriteToHbase.scala" /> <option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/WriteToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/VioViolationToHbase.scala" /> <option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/VioViolationToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/VehicleToHbase.scala" /> <option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/VehicleToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/Test.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala" />
</list> </list>
</option> </option>
</component> </component>
...@@ -196,48 +222,13 @@ ...@@ -196,48 +222,13 @@
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" /> <item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" /> <item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
</path> </path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
<item name="hikcreate" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
<item name="hikcreate" type="462c0819:PsiDirectoryNode" />
<item name="Utils" type="462c0819:PsiDirectoryNode" />
</path>
</expand> </expand>
<select /> <select />
</subPane> </subPane>
</pane> </pane>
<pane id="PackagesPane" />
<pane id="Scope" /> <pane id="Scope" />
<pane id="AndroidView" /> <pane id="AndroidView" />
<pane id="PackagesPane" />
</panes> </panes>
</component> </component>
<component name="PropertiesComponent"> <component name="PropertiesComponent">
...@@ -246,12 +237,13 @@ ...@@ -246,12 +237,13 @@
<property name="Repository.Attach.Sources" value="false" /> <property name="Repository.Attach.Sources" value="false" />
<property name="WebServerToolWindowFactoryState" value="false" /> <property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" /> <property name="aspect.path.notification.shown" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" /> <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" /> <property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="project.structure.last.edited" value="Modules" /> <property name="project.structure.last.edited" value="Modules" />
<property name="project.structure.proportion" value="0.15" /> <property name="project.structure.proportion" value="0.15" />
<property name="project.structure.side.proportion" value="0.2" /> <property name="project.structure.side.proportion" value="0.2" />
<property name="settings.editor.selected.configurable" value="Scala Compiler" /> <property name="settings.editor.selected.configurable" value="vcs.Git" />
</component> </component>
<component name="RunDashboard"> <component name="RunDashboard">
<option name="ruleStates"> <option name="ruleStates">
...@@ -265,7 +257,7 @@ ...@@ -265,7 +257,7 @@
</list> </list>
</option> </option>
</component> </component>
<component name="RunManager" selected="Application.VioViolationToHbase"> <component name="RunManager" selected="Application.Test">
<configuration name="DateFormat" type="Application" factoryName="Application" temporary="true"> <configuration name="DateFormat" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DateFormat" /> <option name="MAIN_CLASS_NAME" value="com.hikcreate.DateFormat" />
<module name="hkys_gy" /> <module name="hkys_gy" />
...@@ -277,6 +269,17 @@ ...@@ -277,6 +269,17 @@
</pattern> </pattern>
</extension> </extension>
</configuration> </configuration>
<configuration name="Test" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DataToHbase.Test" />
<module name="hkys_gy" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.hikcreate.DataToHbase.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration name="VehicleToHbase" type="Application" factoryName="Application" temporary="true"> <configuration name="VehicleToHbase" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DataToHbase.VehicleToHbase" /> <option name="MAIN_CLASS_NAME" value="com.hikcreate.DataToHbase.VehicleToHbase" />
<module name="hkys_gy" /> <module name="hkys_gy" />
...@@ -357,9 +360,11 @@ ...@@ -357,9 +360,11 @@
<item itemvalue="Application.VehicleToHbase" /> <item itemvalue="Application.VehicleToHbase" />
<item itemvalue="Application.VioViolationToHbase" /> <item itemvalue="Application.VioViolationToHbase" />
<item itemvalue="Application.DateFormat" /> <item itemvalue="Application.DateFormat" />
<item itemvalue="Application.Test" />
</list> </list>
<recent_temporary> <recent_temporary>
<list> <list>
<item itemvalue="Application.Test" />
<item itemvalue="Application.VioViolationToHbase" /> <item itemvalue="Application.VioViolationToHbase" />
<item itemvalue="Application.VehicleToHbase" /> <item itemvalue="Application.VehicleToHbase" />
<item itemvalue="Application.WriteToHbase" /> <item itemvalue="Application.WriteToHbase" />
...@@ -379,46 +384,47 @@ ...@@ -379,46 +384,47 @@
<updated>1632388856194</updated> <updated>1632388856194</updated>
<workItem from="1632388859476" duration="3852000" /> <workItem from="1632388859476" duration="3852000" />
<workItem from="1632393132962" duration="82033000" /> <workItem from="1632393132962" duration="82033000" />
<workItem from="1632986444955" duration="3686000" />
<workItem from="1633655451456" duration="18754000" />
</task> </task>
<servers /> <servers />
</component> </component>
<component name="TimeTrackingManager"> <component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="85885000" /> <option name="totallyTimeSpent" value="108325000" />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="-8" y="-8" width="1936" height="1056" extended-state="6" /> <frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
<editor active="true" />
<layout> <layout>
<window_info anchor="right" id="Palette" order="3" /> <window_info anchor="right" id="Palette" order="3" />
<window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.50053304" side_tool="true" weight="0.32897604" />
<window_info anchor="right" id="Maven Projects" order="3" weight="0.7969083" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info id="Capture Tool" order="2" />
<window_info id="Designer" order="2" />
<window_info anchor="right" id="Database" order="3" weight="0.32995737" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" /> <window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Messages" order="7" sideWeight="0.49626866" weight="0.32897604" /> <window_info anchor="bottom" id="Messages" order="7" sideWeight="0.49626866" weight="0.32897604" />
<window_info anchor="right" id="Palette&#9;" order="3" /> <window_info anchor="right" id="Palette&#9;" order="3" />
<window_info id="Image Layers" order="2" /> <window_info id="Image Layers" order="2" />
<window_info anchor="bottom" id="Java Enterprise" order="7" sideWeight="0.49946696" weight="0.32897604" /> <window_info anchor="bottom" id="Java Enterprise" order="7" sideWeight="0.49946696" weight="0.32897604" />
<window_info anchor="right" id="Capture Analysis" order="3" /> <window_info anchor="right" id="Capture Analysis" order="3" />
<window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.4067164" side_tool="true" weight="0.32897604" />
<window_info anchor="right" id="Maven Projects" order="3" weight="0.8160981" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" /> <window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Run" order="2" sideWeight="0.7318763" weight="0.32897604" /> <window_info anchor="bottom" id="Run" order="2" sideWeight="0.5932836" weight="0.32897604" />
<window_info anchor="bottom" id="Terminal" order="7" /> <window_info anchor="bottom" id="Terminal" order="7" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.17643923" /> <window_info id="Capture Tool" order="2" />
<window_info anchor="right" id="Bean Validation" order="3" /> <window_info id="Designer" order="2" />
<window_info anchor="bottom" id="Find" order="1" /> <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.16204691" />
<window_info anchor="right" id="Bean Validation" order="3" weight="0.32995737" />
<window_info anchor="right" id="Database" order="3" weight="0.32995737" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" order="2" />
<window_info anchor="right" id="Theme Preview" order="3" /> <window_info anchor="right" id="Theme Preview" order="3" />
<window_info id="Favorites" order="2" side_tool="true" /> <window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" /> <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" /> <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" /> <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Message" order="0" /> <window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" /> <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="bottom" id="Find" order="1" />
</layout> </layout>
</component> </component>
<component name="TypeScriptGeneratedFilesManager"> <component name="TypeScriptGeneratedFilesManager">
...@@ -429,42 +435,186 @@ ...@@ -429,42 +435,186 @@
</component> </component>
<component name="XDebuggerManager"> <component name="XDebuggerManager">
<breakpoint-manager> <breakpoint-manager>
<option name="time" value="9" /> <breakpoints>
<line-breakpoint enabled="true" type="scala-line">
<url>file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala</url>
<line>51</line>
<properties />
<option name="timeStamp" value="9" />
</line-breakpoint>
</breakpoints>
<option name="time" value="10" />
</breakpoint-manager> </breakpoint-manager>
</component> </component>
<component name="antWorkspaceConfiguration">
<option name="IS_AUTOSCROLL_TO_SOURCE" value="false" />
<option name="FILTER_TARGETS" value="false" />
</component>
<component name="editorHistoryManager"> <component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/pom.xml"> <entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3151">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3933">
<caret line="177" column="18" lean-forward="true" selection-start-line="177" selection-start-column="18" selection-end-line="177" selection-end-column="18" />
<folding>
<element signature="e#35#97#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="7981">
<caret line="354" column="37" selection-start-line="354" selection-start-column="37" selection-end-line="354" selection-end-column="37" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1856#5751#0" />
<element signature="e#5786#10621#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="253">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" />
<folding>
<element signature="e#131#161#0" expanded="true" />
<element signature="e#1977#4126#0" />
<element signature="e#4201#7753#0" />
<element signature="e#7797#11367#0" />
<element signature="e#11463#13112#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3151">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml" />
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="253">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="253">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1856#5751#0" />
<element signature="e#5786#10621#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="414">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" />
<folding>
<element signature="e#131#161#0" expanded="true" />
<element signature="e#1977#4126#0" />
<element signature="e#4201#7753#0" />
<element signature="e#7797#11367#0" />
<element signature="e#11463#13112#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3496"> <state relative-caret-position="3496">
<caret line="152" column="20" lean-forward="true" selection-start-line="152" selection-start-column="20" selection-end-line="152" selection-end-column="20" /> <caret line="152" column="20" lean-forward="true" selection-start-line="152" selection-start-column="20" selection-end-line="152" selection-end-column="20" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml"> <entry file="file://$PROJECT_DIR$/hkys_gy.iml" />
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="230"> <state relative-caret-position="230">
<caret line="13" column="37" lean-forward="true" selection-start-line="14" selection-start-column="53" selection-end-line="14" selection-end-column="72" /> <caret line="13" column="37" lean-forward="true" selection-start-line="14" selection-start-column="53" selection-end-line="14" selection-end-column="72" />
<folding> <folding>
<element signature="e#35#97#0" expanded="true" /> <element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml"> <entry file="file://$PROJECT_DIR$/hkys_gy.iml" />
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hbase/hbase-common/1.1.3/hbase-common-1.1.3.jar!/org/apache/hadoop/hbase/util/Bytes.class"> <entry file="jar://$MAVEN_REPOSITORY$/org/apache/hbase/hbase-common/1.1.3/hbase-common-1.1.3.jar!/org/apache/hadoop/hbase/util/Bytes.class">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="196"> <state relative-caret-position="196">
<caret line="224" selection-start-line="224" selection-end-line="224" /> <caret line="224" selection-start-line="224" selection-end-line="224" />
<folding>
<element signature="e#7651#7652#0" expanded="true" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
...@@ -510,97 +660,98 @@ ...@@ -510,97 +660,98 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/target/classes.timestamp"> <entry file="file://$PROJECT_DIR$/target/classes.timestamp" />
<provider selected="true" editor-type-id="text-editor" /> <entry file="file://$PROJECT_DIR$/target/maven-archiver/pom.properties" />
</entry>
<entry file="file://$PROJECT_DIR$/target/maven-archiver/pom.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="3" column="21" selection-start-line="3" selection-start-column="21" selection-end-line="3" selection-end-column="21" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml"> <entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="152"> <state relative-caret-position="3151">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" /> <caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormatRes.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormatRes.scala" />
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormat.scala" />
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92"> <state relative-caret-position="3933">
<caret line="4" column="33" lean-forward="true" selection-start-line="4" selection-start-column="33" selection-end-line="4" selection-end-column="33" /> <caret line="177" column="18" selection-start-line="177" selection-start-column="18" selection-end-line="177" selection-end-column="18" />
<folding>
<element signature="e#35#97#0" expanded="true" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormat.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="161"> <state relative-caret-position="69">
<caret line="7" column="29" lean-forward="true" selection-start-line="7" selection-start-column="29" selection-end-line="7" selection-end-column="29" /> <caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188"> <state relative-caret-position="92">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" /> <caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding> <folding>
<element signature="e#131#193#0" expanded="true" /> <element signature="e#29#78#0" expanded="true" />
<element signature="e#2032#4181#0" />
<element signature="e#4922#8349#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="179"> <state relative-caret-position="46">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" /> <caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala"> <entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-sql_2.11/2.3.0/spark-sql_2.11-2.3.0.jar!/org/apache/spark/sql/execution/MapElementsExec.class">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188"> <state relative-caret-position="207">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" /> <caret line="9" selection-start-line="9" selection-end-line="9" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#2814#2864#0" />
<element signature="e#6726#11186#0" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala"> <entry file="jar://D:/scala-2.11.8/scala-2.11.8/lib/scala-library.jar!/scala/collection/immutable/StringLike.class">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115"> <state relative-caret-position="277">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" /> <caret line="32" column="6" selection-start-line="32" selection-start-column="6" selection-end-line="32" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/Test.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92"> <state relative-caret-position="667">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" /> <caret line="29" column="4" selection-start-line="29" selection-start-column="4" selection-end-line="29" selection-end-column="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-5368">
<caret line="59" column="6" lean-forward="true" selection-start-line="59" selection-start-column="6" selection-end-line="59" selection-end-column="6" />
<folding> <folding>
<element signature="e#29#62#0" expanded="true" /> <element signature="e#131#161#0" expanded="true" />
<element signature="e#1977#4126#0" />
<element signature="e#4201#7753#0" />
<element signature="e#7797#11367#0" />
<element signature="e#11463#13112#0" />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46"> <state relative-caret-position="1223">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" /> <caret line="340" column="8" selection-start-line="340" selection-start-column="8" selection-end-line="340" selection-end-column="8" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1856#5751#0" />
<element signature="e#5786#10621#0" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
......
hkys_gy/hkys_gy.iml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/Test.scala 0 → 100644
package com.hikcreate.DataToHbase
import org.apache.commons.lang.StringUtils
//import org.apache.spark.sql.catalyst.util.StringUtils
object Test {
def main(args: Array[String]): Unit = {
val str = null
//println(myFun(str))
println(myFun(str))
}
def myFun(str:String) ={
if(StringUtils.isBlank(str)){
""
}else{
str
}
}+
}
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala
...@@ -330,7 +330,7 @@ object VehicleToHbase extends Serializable { ...@@ -330,7 +330,7 @@ object VehicleToHbase extends Serializable {
var columns = datacsvDF.columns var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => { val kvRDD = datacsvDF.flatMap(row => {
val rowkey = Bytes.toBytes(row.getAs[String]("XH")) val rowkey = Bytes.toBytes(row.getAs[String]("xh"))
columns.map( columns.map(
column => { column => {
val colvalue = if (row.getAs[String](column) == null) "null" else row.getAs[String](column) val colvalue = if (row.getAs[String](column) == null) "null" else row.getAs[String](column)
...@@ -353,144 +353,144 @@ object VehicleToHbase extends Serializable { ...@@ -353,144 +353,144 @@ object VehicleToHbase extends Serializable {
}) })
rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration) rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
EsSparkSQL.saveToEs(datacsvDF,"vehicle_gy_ys/doc",Map("es.mapping.id"->"XH")) EsSparkSQL.saveToEs(datacsvDF,"vehicle_gy_ys/doc",Map("es.mapping.id"->"xh"))
sc.stop() sc.stop()
} }
case class Vehicle( case class Vehicle(
XH: String, xh: String,
HPZL: String, hpzl: String,
HPHM: String, hphm: String,
CLPP1: String, clpp1: String,
CLXH: String, clxh: String,
CLPP2: String, clpp2: String,
GCJK: String, gcjk: String,
ZZG: String, zzg: String,
ZZCMC: String, zzcmc: String,
CLSBDH: String, clsbdh: String,
FDJH: String, fdjh: String,
CLLX: String, cllx: String,
CSYS: String, csys: String,
SYXZ: String, syxz: String,
SFZMHM: String, sfzmhm: String,
SFZMMC: String, sfzmmc: String,
SYR: String, syr: String,
SYQ: String, syq: String,
CCDJRQ: String, ccdjrq: String,
DJRQ: String, djrq: String,
YXQZ: String, yxqz: String,
QZBFQZ: String, qzbfqz: String,
FZJG: String, fzjg: String,
GLBM: String, glbm: String,
FPRQ: String, fprq: String,
FZRQ: String, fzrq: String,
FDJRQ: String, fdjrq: String,
FHGZRQ: String, fhgzrq: String,
BXZZRQ: String, bxzzrq: String,
BPCS: String, bpcs: String,
BZCS: String, bzcs: String,
BDJCS: String, bdjcs: String,
DJZSBH: String, djzsbh: String,
ZDJZSHS: String, zdjzshs: String,
DABH: String, dabh: String,
XZQH: String, xzqh: String,
ZT: String, zt: String,
DYBJ: String, dybj: String,
JBR: String, jbr: String,
CLLY: String, clly: String,
LSH: String, lsh: String,
FDJXH: String, fdjxh: String,
RLZL: String, rlzl: String,
PL: String, pl: String,
GL: String, gl: String,
ZXXS: String, zxxs: String,
CWKC: String, cwkc: String,
CWKK: String, cwkk: String,
CWKG: String, cwkg: String,
HXNBCD: String, hxnbcd: String,
HXNBKD: String, hxnbkd: String,
HXNBGD: String, hxnbgd: String,
GBTHPS: String, gbthps: String,
ZS: String, zs: String,
ZJ: String, zj: String,
QLJ: String, qlj: String,
HLJ: String, hlj: String,
LTGG: String, ltgg: String,
LTS: String, lts: String,
ZZL: String, zzl: String,
ZBZL: String, zbzl: String,
HDZZL: String, hdzzl: String,
HDZK: String, hdzk: String,
ZQYZL: String, zqyzl: String,
QPZK: String, qpzk: String,
HPZK: String, hpzk: String,
HBDBQK: String, hbdbqk: String,
CCRQ: String, ccrq: String,
HDFS: String, hdfs: String,
LLPZ1: String, llpz1: String,
PZBH1: String, pzbh1: String,
LLPZ2: String, llpz2: String,
PZBH2: String, pzbh2: String,
XSDW: String, xsdw: String,
XSJG: String, xsjg: String,
XSRQ: String, xsrq: String,
JKPZ: String, jkpz: String,
JKPZHM: String, jkpzhm: String,
HGZBH: String, hgzbh: String,
NSZM: String, nszm: String,
NSZMBH: String, nszmbh: String,
GXRQ: String, gxrq: String,
XGZL: String, xgzl: String,
QMBH: String, qmbh: String,
HMBH: String, hmbh: String,
BZ: String, bz: String,
JYW: String, jyw: String,
ZSXZQH: String, zsxzqh: String,
ZSXXDZ: String, zsxxdz: String,
YZBM1: String, yzbm1: String,
LXDH: String, lxdh: String,
ZZZ: String, zzz: String,
ZZXZQH: String, zzxzqh: String,
ZZXXDZ: String, zzxxdz: String,
YZBM2: String, yzbm2: String,
ZDYZT: String, zdyzt: String,
YXH: String, yxh: String,
CYRY: String, cyry: String,
DPHGZBH: String, dphgzbh: String,
SQDM: String, sqdm: String,
CLYT: String, clyt: String,
YTSX: String, ytsx: String,
DZYX: String, dzyx: String,
XSZBH: String, xszbh: String,
SJHM: String, sjhm: String,
JYHGBZBH: String, jyhgbzbh: String,
DWBH: String, dwbh: String,
SYQSRQ: String, syqsrq: String,
YQJYQZBFQZ: String, yqjyqzbfqz: String,
YQJYQZ2: String, yqjyqz2: String,
FDJGS: String, fdjgs: String,
SFYZHGN: String, sfyzhgn: String,
ZZJGLX: String, zzjglx: String,
WXMBC: String, wxmbc: String,
NCDQSY: String, ncdqsy: String,
HPQYSJ: String, hpqysj: String,
DZBSXLH: String, dzbsxlh: String,
SFXNY: String, sfxny: String,
XNYZL: String, xnyzl: String,
QDDJXH: String, qddjxh: String,
QDDJH: String, qddjh: String,
QDDJGL: String, qddjgl: String,
CNZZZL: String, cnzzzl: String,
CNZZXS: String, cnzzxs: String,
CNZZZDL: String, cnzzzdl: String,
CNZZZDY: String, cnzzzdy: String,
CNZZDTDY: String, cnzzdtdy: String,
HDWJCD: String, hdwjcd: String,
CDQDXSLCGK: String, cdqdxslcgk: String,
CDQDXSLCDS: String, cdqdxslcds: String,
YWJYW: String ywjyw: String
) )
} }
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala
...@@ -7,6 +7,8 @@ package com.hikcreate.DataToHbase ...@@ -7,6 +7,8 @@ package com.hikcreate.DataToHbase
* 数据入库到hbase * 数据入库到hbase
*/ */
import java.text.DecimalFormat
import com.hikcreate.Utils.{DataToHbaseUtils, DateFormatUtils} import com.hikcreate.Utils.{DataToHbaseUtils, DateFormatUtils}
import org.apache.hadoop.hbase.client.Put import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable import org.apache.hadoop.hbase.io.ImmutableBytesWritable
...@@ -29,11 +31,11 @@ object VioViolationToHbase extends Serializable { ...@@ -29,11 +31,11 @@ object VioViolationToHbase extends Serializable {
val conf = new SparkConf() val conf = new SparkConf()
.setAppName("toHbase") .setAppName("toHbase")
.setMaster("local[*]") .setMaster("local[*]")
.set("es.nodes","10.197.236.194,10.197.236.201,10.197.236.193") .set("es.nodes", "10.197.236.194,10.197.236.201,10.197.236.193")
.set("es.port","9200") .set("es.port", "9200")
.set("es.write.operation","index") .set("es.write.operation", "index")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.registerKryoClasses(Array(classOf[ImmutableBytesWritable],classOf[DateFormatUtils])) .registerKryoClasses(Array(classOf[ImmutableBytesWritable], classOf[DateFormatUtils]))
val sc = new SparkContext(conf) val sc = new SparkContext(conf)
val spark = SparkSession.builder().config(conf).getOrCreate() val spark = SparkSession.builder().config(conf).getOrCreate()
...@@ -47,14 +49,14 @@ object VioViolationToHbase extends Serializable { ...@@ -47,14 +49,14 @@ object VioViolationToHbase extends Serializable {
val job: Job = new DataToHbaseUtils().DataToHbaseConf(tablename) val job: Job = new DataToHbaseUtils().DataToHbaseConf(tablename)
val dateUtils=new DateFormatUtils val dateUtils = new DateFormatUtils
import spark.implicits._ import spark.implicits._
val datacsv = spark.read.format("com.databricks.spark.csv") val datacsv = spark.read.format("com.databricks.spark.csv")
.option("header", "true") //在csv第一行有属性"true"，没有就是"false" .option("header", "true") //在csv第一行有属性"true"，没有就是"false"
.option("timestampFormat","yyyy-MM-dd HH:mm:ss") .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
.load("/root/VioViolation.csv") .load("/root/VioViolation.csv")
//.load("C:\\Users\\Administrator\\Documents\\VioViolation.csv") //.load("C:\\Users\\Administrator\\Documents\\VioViolation.csv")
//datacsv.createOrReplaceTempView("VioViolation") //datacsv.createOrReplaceTempView("VioViolation")
...@@ -165,10 +167,9 @@ object VioViolationToHbase extends Serializable { ...@@ -165,10 +167,9 @@ object VioViolationToHbase extends Serializable {
|YLZZ18 |YLZZ18
|from VioViolation |from VioViolation
""".stripMargin)*/ """.stripMargin)*/
//val df=new DecimalFormat("#.00")
//入库到hbase的数据schema映射
val datacsvDF = datacsv.map(r => { val datacsvDF = datacsv.map(r => {
VioViolation( VioViolation(
r.getAs[String]("WFBH"), r.getAs[String]("WFBH"),
...@@ -276,6 +277,222 @@ object VioViolationToHbase extends Serializable { ...@@ -276,6 +277,222 @@ object VioViolationToHbase extends Serializable {
}).toDF() }).toDF()
//入库到es数据schema映射
val datacsvDFLow = datacsv.map(r => {
VioViolationLow(
r.getAs[String]("WFBH"),
r.getAs[String]("JDSLB"),
r.getAs[String]("JDSBH"),
r.getAs[String]("WSJYW"),
r.getAs[String]("RYFL"),
r.getAs[String]("JSZH"),
r.getAs[String]("DABH"),
r.getAs[String]("FZJG"),
r.getAs[String]("ZJCX"),
r.getAs[String]("DSR"),
r.getAs[String]("ZSXZQH"),
r.getAs[String]("ZSXXDZ"),
r.getAs[String]("DH"),
r.getAs[String]("LXFS"),
r.getAs[String]("CLFL"),
r.getAs[String]("HPZL"),
r.getAs[String]("HPHM"),
r.getAs[String]("JDCSYR"),
r.getAs[String]("SYXZ"),
r.getAs[String]("JTFS"),
dateUtils.dateFormatRes(r.getAs[String]("WFSJ")),
r.getAs[String]("XZQH"),
r.getAs[String]("DLLX"),
r.getAs[String]("GLXZDJ"),
r.getAs[String]("WFDD"),
r.getAs[String]("LDDM"),
r.getAs[String]("DDMS"),
r.getAs[String]("DDJDWZ"),
r.getAs[String]("WFDZ"),
r.getAs[String]("WFXW"),
r.getAs[String]("WFJFS"),
r.getAs[String]("FKJE"),
r.getAs[String]("SCZ"),
r.getAs[String]("BZZ"),
r.getAs[String]("ZNJ"),
r.getAs[String]("ZQMJ"),
r.getAs[String]("JKFS"),
r.getAs[String]("FXJG"),
r.getAs[String]("FXJGMC"),
r.getAs[String]("CLJG"),
r.getAs[String]("CLJGMC"),
r.getAs[String]("CFZL"),
dateUtils.dateFormatRes(r.getAs[String]("CLSJ")),
r.getAs[String]("JKBJ"),
dateUtils.dateFormatRes(r.getAs[String]("JKRQ")),
r.getAs[String]("PZBH"),
r.getAs[String]("JSJQBJ"),
r.getAs[String]("JLLX"),
r.getAs[String]("LRR"),
dateUtils.dateFormatRes(r.getAs[String]("LRSJ")),
r.getAs[String]("JBR1"),
r.getAs[String]("JBR2"),
r.getAs[String]("SGDJ"),
r.getAs[String]("CLDXBJ"),
r.getAs[String]("JDCCLDXBJ"),
r.getAs[String]("ZDJLBJ"),
r.getAs[String]("XXLY"),
r.getAs[String]("XRMS"),
r.getAs[String]("DKBJ"),
r.getAs[String]("JMZNJBJ"),
r.getAs[String]("ZDBJ"),
r.getAs[String]("JSJG"),
r.getAs[String]("FSJG"),
dateUtils.dateFormatRes(r.getAs[String]("GXSJ")),
r.getAs[String]("BZ"),
r.getAs[String]("YWJYW"),
r.getAs[String]("ZJMC"),
r.getAs[String]("CCLZRQ"),
r.getAs[String]("NL"),
r.getAs[String]("XB"),
r.getAs[String]("HCBJ"),
r.getAs[String]("JD"),
r.getAs[String]("WD"),
r.getAs[String]("YLZZ1"),
r.getAs[String]("YLZZ2"),
r.getAs[String]("YLZZ3"),
r.getAs[String]("YLZZ4"),
r.getAs[String]("YLZZ5"),
r.getAs[String]("YLZZ6"),
r.getAs[String]("YLZZ7"),
r.getAs[String]("YLZZ8"),
r.getAs[String]("CJFS"),
r.getAs[String]("WFSJ1"),
r.getAs[String]("WFDD1"),
r.getAs[String]("LDDM1"),
r.getAs[String]("DDMS1"),
r.getAs[String]("JSRXZ"),
r.getAs[String]("CLYT"),
r.getAs[String]("XCFW"),
r.getAs[String]("DZZB"),
r.getAs[String]("SFZDRY"),
r.getAs[String]("XYSFZMHM"),
r.getAs[String]("XYXM"),
r.getAs[String]("YLZZ11"),
r.getAs[String]("YLZZ12"),
r.getAs[String]("YLZZ13"),
r.getAs[String]("YLZZ14"),
r.getAs[String]("YLZZ15"),
r.getAs[String]("YLZZ16"),
r.getAs[String]("YLZZ17"),
r.getAs[String]("YLZZ18")
)
}).toDF()
//string字段类型转成long类型
datacsvDFLow.createOrReplaceTempView("violation")
val dataFmt = spark.sql(
"""
|select wfbh,
|jdslb,
|jdsbh,
|wsjyw,
|ryfl,
|jszh,
|dabh,
|fzjg,
|zjcx,
|dsr,
|zsxzqh,
|zsxxdz,
|dh,
|lxfs,
|clfl,
|hpzl,
|hphm,
|jdcsyr,
|syxz,
|jtfs,
|wfsj,
|xzqh,
|dllx,
|glxzdj,
|wfdd,
|lddm,
|ddms,
|ddjdwz,
|wfdz,
|wfxw,
|wfjfs,
|fkje,
|scz,
|bzz,
|cast(znj as Long)znj,
|zqmj,
|jkfs,
|fxjg,
|fxjgmc,
|cljg,
|cljgmc,
|cfzl,
|clsj,
|jkbj,
|jkrq,
|pzbh,
|jsjqbj,
|jllx,
|lrr,
|lrsj,
|jbr1,
|jbr2,
|sgdj,
|cldxbj,
|jdccldxbj,
|zdjlbj,
|xxly,
|xrms,
|dkbj,
|jmznjbj,
|zdbj,
|jsjg,
|fsjg,
|gxsj,
|bz,
|ywjyw,
|zjmc,
|cclzrq,
|nl,
|xb,
|hcbj,
|jd,
|wd,
|ylzz1,
|ylzz2,
|ylzz3,
|ylzz4,
|ylzz5,
|ylzz6,
|ylzz7,
|ylzz8,
|cjfs,
|wfsj1,
|wfdd1,
|lddm1,
|ddms1,
|jsrxz,
|clyt,
|xcfw,
|dzzb,
|sfzdry,
|xysfzmhm,
|xyxm,
|ylzz11,
|ylzz12,
|ylzz13,
|ylzz14,
|ylzz15,
|ylzz16,
|ylzz17,
|ylzz18
|from violation
""".stripMargin)
var columns = datacsvDF.columns var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => { val kvRDD = datacsvDF.flatMap(row => {
val rowkey = Bytes.toBytes(row.getAs[String]("WFBH")) val rowkey = Bytes.toBytes(row.getAs[String]("WFBH"))
...@@ -300,10 +517,10 @@ object VioViolationToHbase extends Serializable { ...@@ -300,10 +517,10 @@ object VioViolationToHbase extends Serializable {
}) })
rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration) rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
EsSparkSQL.saveToEs(datacsvDF,"violation_ys/doc",Map("es.mapping.id"->"WFBH")) EsSparkSQL.saveToEs(dataFmt,"violation_ys/doc",Map("es.mapping.id"->"wfbh"))
sc.stop() sc.stop()
} }
case class VioViolation(WFBH: String, case class VioViolation(WFBH: String,
JDSLB: String, JDSLB: String,
JDSBH: String, JDSBH: String,
...@@ -405,4 +622,107 @@ object VioViolationToHbase extends Serializable { ...@@ -405,4 +622,107 @@ object VioViolationToHbase extends Serializable {
YLZZ16: String, YLZZ16: String,
YLZZ17: String, YLZZ17: String,
YLZZ18: String) YLZZ18: String)
case class VioViolationLow(wfbh: String,
jdslb: String,
jdsbh: String,
wsjyw: String,
ryfl: String,
jszh: String,
dabh: String,
fzjg: String,
zjcx: String,
dsr: String,
zsxzqh: String,
zsxxdz: String,
dh: String,
lxfs: String,
clfl: String,
hpzl: String,
hphm: String,
jdcsyr: String,
syxz: String,
jtfs: String,
wfsj: String,
xzqh: String,
dllx: String,
glxzdj: String,
wfdd: String,
lddm: String,
ddms: String,
ddjdwz: String,
wfdz: String,
wfxw: String,
wfjfs: String,
fkje: String,
scz: String,
bzz: String,
znj: String,
zqmj: String,
jkfs: String,
fxjg: String,
fxjgmc: String,
cljg: String,
cljgmc: String,
cfzl: String,
clsj: String,
jkbj: String,
jkrq: String,
pzbh: String,
jsjqbj: String,
jllx: String,
lrr: String,
lrsj: String,
jbr1: String,
jbr2: String,
sgdj: String,
cldxbj: String,
jdccldxbj: String,
zdjlbj: String,
xxly: String,
xrms: String,
dkbj: String,
jmznjbj: String,
zdbj: String,
jsjg: String,
fsjg: String,
gxsj: String,
bz: String,
ywjyw: String,
zjmc: String,
cclzrq: String,
nl: String,
xb: String,
hcbj: String,
jd: String,
wd: String,
ylzz1: String,
ylzz2: String,
ylzz3: String,
ylzz4: String,
ylzz5: String,
ylzz6: String,
ylzz7: String,
ylzz8: String,
cjfs: String,
wfsj1: String,
wfdd1: String,
lddm1: String,
ddms1: String,
jsrxz: String,
clyt: String,
xcfw: String,
dzzb: String,
sfzdry: String,
xysfzmhm: String,
xyxm: String,
ylzz11: String,
ylzz12: String,
ylzz13: String,
ylzz14: String,
ylzz15: String,
ylzz16: String,
ylzz17: String,
ylzz18: String)
} }
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala
...@@ -170,7 +170,7 @@ object WriteToHbase extends Serializable { ...@@ -170,7 +170,7 @@ object WriteToHbase extends Serializable {
r.getAs[String]("RYZT") r.getAs[String]("RYZT")
) )
}).toDF() }).toDF()
datacsvDF.show()
var columns = datacsvDF.columns var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => { val kvRDD = datacsvDF.flatMap(row => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment