Skip to content

D
datatoHbaseAdEs
Commit fac63730 by wangkai
Options

字段类型转换和schema大小写转换

parent 0134edf0
Showing with 892 additions and 391 deletions
hkys_gy/.idea/workspace.xml
......@@ -30,26 +30,19 @@
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="152">
<state relative-caret-position="3151">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="hkys_gy.iml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/hkys_gy.iml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file leaf-file-name="WriteToHbase.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="179">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<state relative-caret-position="3933">
<caret line="177" column="18" selection-start-line="177" selection-start-column="18" selection-end-line="177" selection-end-column="18" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding>
</state>
</provider>
......@@ -58,7 +51,7 @@
<file leaf-file-name="DateUtils.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<state relative-caret-position="69">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
......@@ -67,15 +60,15 @@
</provider>
</entry>
</file>
<file leaf-file-name="VehicleToHbase.scala" pinned="false" current-in-tab="false">
<file leaf-file-name="VehicleToHbase.scala" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<state relative-caret-position="1223">
<caret line="340" column="8" selection-start-line="340" selection-start-column="8" selection-end-line="340" selection-end-column="8" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#2814#2864#0" />
<element signature="e#6726#11186#0" />
<element signature="e#1856#5751#0" />
<element signature="e#5786#10621#0" />
</folding>
</state>
</provider>
......@@ -84,18 +77,47 @@
<file leaf-file-name="VioViolationToHbase.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" />
<state relative-caret-position="-5368">
<caret line="59" column="6" lean-forward="true" selection-start-line="59" selection-start-column="6" selection-end-line="59" selection-end-column="6" />
<folding>
<element signature="e#131#193#0" expanded="true" />
<element signature="e#2032#4181#0" />
<element signature="e#4922#8349#0" />
<element signature="e#131#161#0" expanded="true" />
<element signature="e#1977#4126#0" />
<element signature="e#4201#7753#0" />
<element signature="e#7797#11367#0" />
<element signature="e#11463#13112#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DateFormatUtils.scala" pinned="false" current-in-tab="true">
<file leaf-file-name="Test.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/Test.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="667">
<caret line="29" column="4" selection-start-line="29" selection-start-column="4" selection-end-line="29" selection-end-column="4" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="StringLike.class" pinned="false" current-in-tab="false">
<entry file="jar://D:/scala-2.11.8/scala-2.11.8/lib/scala-library.jar!/scala/collection/immutable/StringLike.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="277">
<caret line="32" column="6" selection-start-line="32" selection-start-column="6" selection-end-line="32" selection-end-column="6" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="MapElementsExec.class" pinned="false" current-in-tab="false">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-sql_2.11/2.3.0/spark-sql_2.11-2.3.0.jar!/org/apache/spark/sql/execution/MapElementsExec.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
<caret line="9" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DateFormatUtils.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
......@@ -107,7 +129,7 @@
<file leaf-file-name="DataToHbaseUtils.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<state relative-caret-position="92">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
......@@ -138,6 +160,10 @@
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/WriteToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/VioViolationToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/VehicleToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/Test.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala" />
</list>
</option>
</component>
......@@ -196,48 +222,13 @@
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
<item name="hikcreate" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
<item name="hikcreate" type="462c0819:PsiDirectoryNode" />
<item name="Utils" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="PackagesPane" />
<pane id="Scope" />
<pane id="AndroidView" />
<pane id="PackagesPane" />
</panes>
</component>
<component name="PropertiesComponent">
......@@ -246,12 +237,13 @@
<property name="Repository.Attach.Sources" value="false" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="project.structure.last.edited" value="Modules" />
<property name="project.structure.proportion" value="0.15" />
<property name="project.structure.side.proportion" value="0.2" />
<property name="settings.editor.selected.configurable" value="Scala Compiler" />
<property name="settings.editor.selected.configurable" value="vcs.Git" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
......@@ -265,7 +257,7 @@
</list>
</option>
</component>
<component name="RunManager" selected="Application.VioViolationToHbase">
<component name="RunManager" selected="Application.Test">
<configuration name="DateFormat" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DateFormat" />
<module name="hkys_gy" />
......@@ -277,6 +269,17 @@
</pattern>
</extension>
</configuration>
<configuration name="Test" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DataToHbase.Test" />
<module name="hkys_gy" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.hikcreate.DataToHbase.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration name="VehicleToHbase" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DataToHbase.VehicleToHbase" />
<module name="hkys_gy" />
......@@ -357,9 +360,11 @@
<item itemvalue="Application.VehicleToHbase" />
<item itemvalue="Application.VioViolationToHbase" />
<item itemvalue="Application.DateFormat" />
<item itemvalue="Application.Test" />
</list>
<recent_temporary>
<list>
<item itemvalue="Application.Test" />
<item itemvalue="Application.VioViolationToHbase" />
<item itemvalue="Application.VehicleToHbase" />
<item itemvalue="Application.WriteToHbase" />
......@@ -379,46 +384,47 @@
<updated>1632388856194</updated>
<workItem from="1632388859476" duration="3852000" />
<workItem from="1632393132962" duration="82033000" />
<workItem from="1632986444955" duration="3686000" />
<workItem from="1633655451456" duration="18754000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="85885000" />
<option name="totallyTimeSpent" value="108325000" />
</component>
<component name="ToolWindowManager">
<frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
<editor active="true" />
<layout>
<window_info anchor="right" id="Palette" order="3" />
<window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.50053304" side_tool="true" weight="0.32897604" />
<window_info anchor="right" id="Maven Projects" order="3" weight="0.7969083" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info id="Capture Tool" order="2" />
<window_info id="Designer" order="2" />
<window_info anchor="right" id="Database" order="3" weight="0.32995737" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Messages" order="7" sideWeight="0.49626866" weight="0.32897604" />
<window_info anchor="right" id="Palette&#9;" order="3" />
<window_info id="Image Layers" order="2" />
<window_info anchor="bottom" id="Java Enterprise" order="7" sideWeight="0.49946696" weight="0.32897604" />
<window_info anchor="right" id="Capture Analysis" order="3" />
<window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.4067164" side_tool="true" weight="0.32897604" />
<window_info anchor="right" id="Maven Projects" order="3" weight="0.8160981" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Run" order="2" sideWeight="0.7318763" weight="0.32897604" />
<window_info anchor="bottom" id="Run" order="2" sideWeight="0.5932836" weight="0.32897604" />
<window_info anchor="bottom" id="Terminal" order="7" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.17643923" />
<window_info anchor="right" id="Bean Validation" order="3" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info id="Capture Tool" order="2" />
<window_info id="Designer" order="2" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.16204691" />
<window_info anchor="right" id="Bean Validation" order="3" weight="0.32995737" />
<window_info anchor="right" id="Database" order="3" weight="0.32995737" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" order="2" />
<window_info anchor="right" id="Theme Preview" order="3" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="bottom" id="Find" order="1" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
......@@ -429,42 +435,186 @@
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<option name="time" value="9" />
<breakpoints>
<line-breakpoint enabled="true" type="scala-line">
<url>file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala</url>
<line>51</line>
<properties />
<option name="timeStamp" value="9" />
</line-breakpoint>
</breakpoints>
<option name="time" value="10" />
</breakpoint-manager>
</component>
<component name="antWorkspaceConfiguration">
<option name="IS_AUTOSCROLL_TO_SOURCE" value="false" />
<option name="FILTER_TARGETS" value="false" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3151">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3933">
<caret line="177" column="18" lean-forward="true" selection-start-line="177" selection-start-column="18" selection-end-line="177" selection-end-column="18" />
<folding>
<element signature="e#35#97#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="7981">
<caret line="354" column="37" selection-start-line="354" selection-start-column="37" selection-end-line="354" selection-end-column="37" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1856#5751#0" />
<element signature="e#5786#10621#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="253">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" />
<folding>
<element signature="e#131#161#0" expanded="true" />
<element signature="e#1977#4126#0" />
<element signature="e#4201#7753#0" />
<element signature="e#7797#11367#0" />
<element signature="e#11463#13112#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3151">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml" />
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="253">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="253">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1856#5751#0" />
<element signature="e#5786#10621#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="414">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" />
<folding>
<element signature="e#131#161#0" expanded="true" />
<element signature="e#1977#4126#0" />
<element signature="e#4201#7753#0" />
<element signature="e#7797#11367#0" />
<element signature="e#11463#13112#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3496">
<caret line="152" column="20" lean-forward="true" selection-start-line="152" selection-start-column="20" selection-end-line="152" selection-end-column="20" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml" />
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="230">
<caret line="13" column="37" lean-forward="true" selection-start-line="14" selection-start-column="53" selection-end-line="14" selection-end-column="72" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml" />
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hbase/hbase-common/1.1.3/hbase-common-1.1.3.jar!/org/apache/hadoop/hbase/util/Bytes.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="196">
<caret line="224" selection-start-line="224" selection-end-line="224" />
<folding>
<element signature="e#7651#7652#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
......@@ -510,97 +660,98 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/target/classes.timestamp">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/target/maven-archiver/pom.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="3" column="21" selection-start-line="3" selection-start-column="21" selection-end-line="3" selection-end-column="21" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/target/classes.timestamp" />
<entry file="file://$PROJECT_DIR$/target/maven-archiver/pom.properties" />
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="152">
<state relative-caret-position="3151">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormatRes.scala">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormatRes.scala" />
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormat.scala" />
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="4" column="33" lean-forward="true" selection-start-line="4" selection-start-column="33" selection-end-line="4" selection-end-column="33" />
<state relative-caret-position="3933">
<caret line="177" column="18" selection-start-line="177" selection-start-column="18" selection-end-line="177" selection-end-column="18" />
<folding>
<element signature="e#35#97#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormat.scala">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="161">
<caret line="7" column="29" lean-forward="true" selection-start-line="7" selection-start-column="29" selection-end-line="7" selection-end-column="29" />
<state relative-caret-position="69">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" />
<state relative-caret-position="92">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#131#193#0" expanded="true" />
<element signature="e#2032#4181#0" />
<element signature="e#4922#8349#0" />
<element signature="e#29#78#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="179">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding>
<state relative-caret-position="46">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-sql_2.11/2.3.0/spark-sql_2.11-2.3.0.jar!/org/apache/spark/sql/execution/MapElementsExec.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#2814#2864#0" />
<element signature="e#6726#11186#0" />
</folding>
<state relative-caret-position="207">
<caret line="9" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<entry file="jar://D:/scala-2.11.8/scala-2.11.8/lib/scala-library.jar!/scala/collection/immutable/StringLike.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
</folding>
<state relative-caret-position="277">
<caret line="32" column="6" selection-start-line="32" selection-start-column="6" selection-end-line="32" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/Test.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<state relative-caret-position="667">
<caret line="29" column="4" selection-start-line="29" selection-start-column="4" selection-end-line="29" selection-end-column="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-5368">
<caret line="59" column="6" lean-forward="true" selection-start-line="59" selection-start-column="6" selection-end-line="59" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
<element signature="e#131#161#0" expanded="true" />
<element signature="e#1977#4126#0" />
<element signature="e#4201#7753#0" />
<element signature="e#7797#11367#0" />
<element signature="e#11463#13112#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
<state relative-caret-position="1223">
<caret line="340" column="8" selection-start-line="340" selection-start-column="8" selection-end-line="340" selection-end-column="8" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1856#5751#0" />
<element signature="e#5786#10621#0" />
</folding>
</state>
</provider>
</entry>
......
hkys_gy/hkys_gy.iml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/Test.scala 0 → 100644
package com.hikcreate.DataToHbase
import org.apache.commons.lang.StringUtils
//import org.apache.spark.sql.catalyst.util.StringUtils
object Test {
def main(args: Array[String]): Unit = {
val str = null
//println(myFun(str))
println(myFun(str))
}
def myFun(str:String) ={
if(StringUtils.isBlank(str)){
""
}else{
str
}
}+
}
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala
......@@ -47,7 +47,7 @@ object VehicleToHbase extends Serializable {
val datacsv = spark.read.format("com.databricks.spark.csv")
.option("header", "true") //在csv第一行有属性"true"，没有就是"false"
.option("timestampFormat","yyyy-MM-dd HH:mm:ss")
.load("/root/vehicle.csv")
.load("/root/vehicle.csv")
//.load("C:\\Users\\Administrator\\Documents\\vehicle.csv")
//datacsv.createOrReplaceTempView("vehicle")
/*val datafmt=spark.sql(
......@@ -330,7 +330,7 @@ object VehicleToHbase extends Serializable {
var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => {
val rowkey = Bytes.toBytes(row.getAs[String]("XH"))
val rowkey = Bytes.toBytes(row.getAs[String]("xh"))
columns.map(
column => {
val colvalue = if (row.getAs[String](column) == null) "null" else row.getAs[String](column)
......@@ -353,144 +353,144 @@ object VehicleToHbase extends Serializable {
})
rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
EsSparkSQL.saveToEs(datacsvDF,"vehicle_gy_ys/doc",Map("es.mapping.id"->"XH"))
EsSparkSQL.saveToEs(datacsvDF,"vehicle_gy_ys/doc",Map("es.mapping.id"->"xh"))
sc.stop()
}
case class Vehicle(
XH: String,
HPZL: String,
HPHM: String,
CLPP1: String,
CLXH: String,
CLPP2: String,
GCJK: String,
ZZG: String,
ZZCMC: String,
CLSBDH: String,
FDJH: String,
CLLX: String,
CSYS: String,
SYXZ: String,
SFZMHM: String,
SFZMMC: String,
SYR: String,
SYQ: String,
CCDJRQ: String,
DJRQ: String,
YXQZ: String,
QZBFQZ: String,
FZJG: String,
GLBM: String,
FPRQ: String,
FZRQ: String,
FDJRQ: String,
FHGZRQ: String,
BXZZRQ: String,
BPCS: String,
BZCS: String,
BDJCS: String,
DJZSBH: String,
ZDJZSHS: String,
DABH: String,
XZQH: String,
ZT: String,
DYBJ: String,
JBR: String,
CLLY: String,
LSH: String,
FDJXH: String,
RLZL: String,
PL: String,
GL: String,
ZXXS: String,
CWKC: String,
CWKK: String,
CWKG: String,
HXNBCD: String,
HXNBKD: String,
HXNBGD: String,
GBTHPS: String,
ZS: String,
ZJ: String,
QLJ: String,
HLJ: String,
LTGG: String,
LTS: String,
ZZL: String,
ZBZL: String,
HDZZL: String,
HDZK: String,
ZQYZL: String,
QPZK: String,
HPZK: String,
HBDBQK: String,
CCRQ: String,
HDFS: String,
LLPZ1: String,
PZBH1: String,
LLPZ2: String,
PZBH2: String,
XSDW: String,
XSJG: String,
XSRQ: String,
JKPZ: String,
JKPZHM: String,
HGZBH: String,
NSZM: String,
NSZMBH: String,
GXRQ: String,
XGZL: String,
QMBH: String,
HMBH: String,
BZ: String,
JYW: String,
ZSXZQH: String,
ZSXXDZ: String,
YZBM1: String,
LXDH: String,
ZZZ: String,
ZZXZQH: String,
ZZXXDZ: String,
YZBM2: String,
ZDYZT: String,
YXH: String,
CYRY: String,
DPHGZBH: String,
SQDM: String,
CLYT: String,
YTSX: String,
DZYX: String,
XSZBH: String,
SJHM: String,
JYHGBZBH: String,
DWBH: String,
SYQSRQ: String,
YQJYQZBFQZ: String,
YQJYQZ2: String,
FDJGS: String,
SFYZHGN: String,
ZZJGLX: String,
WXMBC: String,
NCDQSY: String,
HPQYSJ: String,
DZBSXLH: String,
SFXNY: String,
XNYZL: String,
QDDJXH: String,
QDDJH: String,
QDDJGL: String,
CNZZZL: String,
CNZZXS: String,
CNZZZDL: String,
CNZZZDY: String,
CNZZDTDY: String,
HDWJCD: String,
CDQDXSLCGK: String,
CDQDXSLCDS: String,
YWJYW: String
xh: String,
hpzl: String,
hphm: String,
clpp1: String,
clxh: String,
clpp2: String,
gcjk: String,
zzg: String,
zzcmc: String,
clsbdh: String,
fdjh: String,
cllx: String,
csys: String,
syxz: String,
sfzmhm: String,
sfzmmc: String,
syr: String,
syq: String,
ccdjrq: String,
djrq: String,
yxqz: String,
qzbfqz: String,
fzjg: String,
glbm: String,
fprq: String,
fzrq: String,
fdjrq: String,
fhgzrq: String,
bxzzrq: String,
bpcs: String,
bzcs: String,
bdjcs: String,
djzsbh: String,
zdjzshs: String,
dabh: String,
xzqh: String,
zt: String,
dybj: String,
jbr: String,
clly: String,
lsh: String,
fdjxh: String,
rlzl: String,
pl: String,
gl: String,
zxxs: String,
cwkc: String,
cwkk: String,
cwkg: String,
hxnbcd: String,
hxnbkd: String,
hxnbgd: String,
gbthps: String,
zs: String,
zj: String,
qlj: String,
hlj: String,
ltgg: String,
lts: String,
zzl: String,
zbzl: String,
hdzzl: String,
hdzk: String,
zqyzl: String,
qpzk: String,
hpzk: String,
hbdbqk: String,
ccrq: String,
hdfs: String,
llpz1: String,
pzbh1: String,
llpz2: String,
pzbh2: String,
xsdw: String,
xsjg: String,
xsrq: String,
jkpz: String,
jkpzhm: String,
hgzbh: String,
nszm: String,
nszmbh: String,
gxrq: String,
xgzl: String,
qmbh: String,
hmbh: String,
bz: String,
jyw: String,
zsxzqh: String,
zsxxdz: String,
yzbm1: String,
lxdh: String,
zzz: String,
zzxzqh: String,
zzxxdz: String,
yzbm2: String,
zdyzt: String,
yxh: String,
cyry: String,
dphgzbh: String,
sqdm: String,
clyt: String,
ytsx: String,
dzyx: String,
xszbh: String,
sjhm: String,
jyhgbzbh: String,
dwbh: String,
syqsrq: String,
yqjyqzbfqz: String,
yqjyqz2: String,
fdjgs: String,
sfyzhgn: String,
zzjglx: String,
wxmbc: String,
ncdqsy: String,
hpqysj: String,
dzbsxlh: String,
sfxny: String,
xnyzl: String,
qddjxh: String,
qddjh: String,
qddjgl: String,
cnzzzl: String,
cnzzxs: String,
cnzzzdl: String,
cnzzzdy: String,
cnzzdtdy: String,
hdwjcd: String,
cdqdxslcgk: String,
cdqdxslcds: String,
ywjyw: String
)
}
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala
......@@ -7,6 +7,8 @@ package com.hikcreate.DataToHbase
* 数据入库到hbase
*/
import java.text.DecimalFormat
import com.hikcreate.Utils.{DataToHbaseUtils, DateFormatUtils}
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
......@@ -29,11 +31,11 @@ object VioViolationToHbase extends Serializable {
val conf = new SparkConf()
.setAppName("toHbase")
.setMaster("local[*]")
.set("es.nodes","10.197.236.194,10.197.236.201,10.197.236.193")
.set("es.port","9200")
.set("es.write.operation","index")
.set("es.nodes", "10.197.236.194,10.197.236.201,10.197.236.193")
.set("es.port", "9200")
.set("es.write.operation", "index")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.registerKryoClasses(Array(classOf[ImmutableBytesWritable],classOf[DateFormatUtils]))
.registerKryoClasses(Array(classOf[ImmutableBytesWritable], classOf[DateFormatUtils]))
val sc = new SparkContext(conf)
val spark = SparkSession.builder().config(conf).getOrCreate()
......@@ -47,14 +49,14 @@ object VioViolationToHbase extends Serializable {
val job: Job = new DataToHbaseUtils().DataToHbaseConf(tablename)
val dateUtils=new DateFormatUtils
val dateUtils = new DateFormatUtils
import spark.implicits._
val datacsv = spark.read.format("com.databricks.spark.csv")
.option("header", "true") //在csv第一行有属性"true"，没有就是"false"
.option("timestampFormat","yyyy-MM-dd HH:mm:ss")
.option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
.load("/root/VioViolation.csv")
//.load("C:\\Users\\Administrator\\Documents\\VioViolation.csv")
//datacsv.createOrReplaceTempView("VioViolation")
......@@ -165,10 +167,9 @@ object VioViolationToHbase extends Serializable {
|YLZZ18
|from VioViolation
""".stripMargin)*/
//val df=new DecimalFormat("#.00")
//入库到hbase的数据schema映射
val datacsvDF = datacsv.map(r => {
VioViolation(
r.getAs[String]("WFBH"),
......@@ -276,6 +277,222 @@ object VioViolationToHbase extends Serializable {
}).toDF()
//入库到es数据schema映射
val datacsvDFLow = datacsv.map(r => {
VioViolationLow(
r.getAs[String]("WFBH"),
r.getAs[String]("JDSLB"),
r.getAs[String]("JDSBH"),
r.getAs[String]("WSJYW"),
r.getAs[String]("RYFL"),
r.getAs[String]("JSZH"),
r.getAs[String]("DABH"),
r.getAs[String]("FZJG"),
r.getAs[String]("ZJCX"),
r.getAs[String]("DSR"),
r.getAs[String]("ZSXZQH"),
r.getAs[String]("ZSXXDZ"),
r.getAs[String]("DH"),
r.getAs[String]("LXFS"),
r.getAs[String]("CLFL"),
r.getAs[String]("HPZL"),
r.getAs[String]("HPHM"),
r.getAs[String]("JDCSYR"),
r.getAs[String]("SYXZ"),
r.getAs[String]("JTFS"),
dateUtils.dateFormatRes(r.getAs[String]("WFSJ")),
r.getAs[String]("XZQH"),
r.getAs[String]("DLLX"),
r.getAs[String]("GLXZDJ"),
r.getAs[String]("WFDD"),
r.getAs[String]("LDDM"),
r.getAs[String]("DDMS"),
r.getAs[String]("DDJDWZ"),
r.getAs[String]("WFDZ"),
r.getAs[String]("WFXW"),
r.getAs[String]("WFJFS"),
r.getAs[String]("FKJE"),
r.getAs[String]("SCZ"),
r.getAs[String]("BZZ"),
r.getAs[String]("ZNJ"),
r.getAs[String]("ZQMJ"),
r.getAs[String]("JKFS"),
r.getAs[String]("FXJG"),
r.getAs[String]("FXJGMC"),
r.getAs[String]("CLJG"),
r.getAs[String]("CLJGMC"),
r.getAs[String]("CFZL"),
dateUtils.dateFormatRes(r.getAs[String]("CLSJ")),
r.getAs[String]("JKBJ"),
dateUtils.dateFormatRes(r.getAs[String]("JKRQ")),
r.getAs[String]("PZBH"),
r.getAs[String]("JSJQBJ"),
r.getAs[String]("JLLX"),
r.getAs[String]("LRR"),
dateUtils.dateFormatRes(r.getAs[String]("LRSJ")),
r.getAs[String]("JBR1"),
r.getAs[String]("JBR2"),
r.getAs[String]("SGDJ"),
r.getAs[String]("CLDXBJ"),
r.getAs[String]("JDCCLDXBJ"),
r.getAs[String]("ZDJLBJ"),
r.getAs[String]("XXLY"),
r.getAs[String]("XRMS"),
r.getAs[String]("DKBJ"),
r.getAs[String]("JMZNJBJ"),
r.getAs[String]("ZDBJ"),
r.getAs[String]("JSJG"),
r.getAs[String]("FSJG"),
dateUtils.dateFormatRes(r.getAs[String]("GXSJ")),
r.getAs[String]("BZ"),
r.getAs[String]("YWJYW"),
r.getAs[String]("ZJMC"),
r.getAs[String]("CCLZRQ"),
r.getAs[String]("NL"),
r.getAs[String]("XB"),
r.getAs[String]("HCBJ"),
r.getAs[String]("JD"),
r.getAs[String]("WD"),
r.getAs[String]("YLZZ1"),
r.getAs[String]("YLZZ2"),
r.getAs[String]("YLZZ3"),
r.getAs[String]("YLZZ4"),
r.getAs[String]("YLZZ5"),
r.getAs[String]("YLZZ6"),
r.getAs[String]("YLZZ7"),
r.getAs[String]("YLZZ8"),
r.getAs[String]("CJFS"),
r.getAs[String]("WFSJ1"),
r.getAs[String]("WFDD1"),
r.getAs[String]("LDDM1"),
r.getAs[String]("DDMS1"),
r.getAs[String]("JSRXZ"),
r.getAs[String]("CLYT"),
r.getAs[String]("XCFW"),
r.getAs[String]("DZZB"),
r.getAs[String]("SFZDRY"),
r.getAs[String]("XYSFZMHM"),
r.getAs[String]("XYXM"),
r.getAs[String]("YLZZ11"),
r.getAs[String]("YLZZ12"),
r.getAs[String]("YLZZ13"),
r.getAs[String]("YLZZ14"),
r.getAs[String]("YLZZ15"),
r.getAs[String]("YLZZ16"),
r.getAs[String]("YLZZ17"),
r.getAs[String]("YLZZ18")
)
}).toDF()
//string字段类型转成long类型
datacsvDFLow.createOrReplaceTempView("violation")
val dataFmt = spark.sql(
"""
|select wfbh,
|jdslb,
|jdsbh,
|wsjyw,
|ryfl,
|jszh,
|dabh,
|fzjg,
|zjcx,
|dsr,
|zsxzqh,
|zsxxdz,
|dh,
|lxfs,
|clfl,
|hpzl,
|hphm,
|jdcsyr,
|syxz,
|jtfs,
|wfsj,
|xzqh,
|dllx,
|glxzdj,
|wfdd,
|lddm,
|ddms,
|ddjdwz,
|wfdz,
|wfxw,
|wfjfs,
|fkje,
|scz,
|bzz,
|cast(znj as Long)znj,
|zqmj,
|jkfs,
|fxjg,
|fxjgmc,
|cljg,
|cljgmc,
|cfzl,
|clsj,
|jkbj,
|jkrq,
|pzbh,
|jsjqbj,
|jllx,
|lrr,
|lrsj,
|jbr1,
|jbr2,
|sgdj,
|cldxbj,
|jdccldxbj,
|zdjlbj,
|xxly,
|xrms,
|dkbj,
|jmznjbj,
|zdbj,
|jsjg,
|fsjg,
|gxsj,
|bz,
|ywjyw,
|zjmc,
|cclzrq,
|nl,
|xb,
|hcbj,
|jd,
|wd,
|ylzz1,
|ylzz2,
|ylzz3,
|ylzz4,
|ylzz5,
|ylzz6,
|ylzz7,
|ylzz8,
|cjfs,
|wfsj1,
|wfdd1,
|lddm1,
|ddms1,
|jsrxz,
|clyt,
|xcfw,
|dzzb,
|sfzdry,
|xysfzmhm,
|xyxm,
|ylzz11,
|ylzz12,
|ylzz13,
|ylzz14,
|ylzz15,
|ylzz16,
|ylzz17,
|ylzz18
|from violation
""".stripMargin)
var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => {
val rowkey = Bytes.toBytes(row.getAs[String]("WFBH"))
......@@ -300,109 +517,212 @@ object VioViolationToHbase extends Serializable {
})
rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
EsSparkSQL.saveToEs(datacsvDF,"violation_ys/doc",Map("es.mapping.id"->"WFBH"))
EsSparkSQL.saveToEs(dataFmt,"violation_ys/doc",Map("es.mapping.id"->"wfbh"))
sc.stop()
}
case class VioViolation(WFBH: String,
JDSLB: String,
JDSBH: String,
WSJYW: String,
RYFL: String,
JSZH: String,
DABH: String,
FZJG: String,
ZJCX: String,
DSR: String,
ZSXZQH: String,
ZSXXDZ: String,
DH: String,
LXFS: String,
CLFL: String,
HPZL: String,
HPHM: String,
JDCSYR: String,
SYXZ: String,
JTFS: String,
WFSJ: String,
XZQH: String,
DLLX: String,
GLXZDJ: String,
WFDD: String,
LDDM: String,
DDMS: String,
DDJDWZ: String,
WFDZ: String,
WFXW: String,
WFJFS: String,
FKJE: String,
SCZ: String,
BZZ: String,
ZNJ: String,
ZQMJ: String,
JKFS: String,
FXJG: String,
FXJGMC: String,
CLJG: String,
CLJGMC: String,
CFZL: String,
CLSJ: String,
JKBJ: String,
JKRQ: String,
PZBH: String,
JSJQBJ: String,
JLLX: String,
LRR: String,
LRSJ: String,
JBR1: String,
JBR2: String,
SGDJ: String,
CLDXBJ: String,
JDCCLDXBJ: String,
ZDJLBJ: String,
XXLY: String,
XRMS: String,
DKBJ: String,
JMZNJBJ: String,
ZDBJ: String,
JSJG: String,
FSJG: String,
GXSJ: String,
BZ: String,
YWJYW: String,
ZJMC: String,
CCLZRQ: String,
NL: String,
XB: String,
HCBJ: String,
JD: String,
WD: String,
YLZZ1: String,
YLZZ2: String,
YLZZ3: String,
YLZZ4: String,
YLZZ5: String,
YLZZ6: String,
YLZZ7: String,
YLZZ8: String,
CJFS: String,
WFSJ1: String,
WFDD1: String,
LDDM1: String,
DDMS1: String,
JSRXZ: String,
CLYT: String,
XCFW: String,
DZZB: String,
SFZDRY: String,
XYSFZMHM: String,
XYXM: String,
YLZZ11: String,
YLZZ12: String,
YLZZ13: String,
YLZZ14: String,
YLZZ15: String,
YLZZ16: String,
YLZZ17: String,
YLZZ18: String)
case class VioViolation(WFBH: String,
JDSLB: String,
JDSBH: String,
WSJYW: String,
RYFL: String,
JSZH: String,
DABH: String,
FZJG: String,
ZJCX: String,
DSR: String,
ZSXZQH: String,
ZSXXDZ: String,
DH: String,
LXFS: String,
CLFL: String,
HPZL: String,
HPHM: String,
JDCSYR: String,
SYXZ: String,
JTFS: String,
WFSJ: String,
XZQH: String,
DLLX: String,
GLXZDJ: String,
WFDD: String,
LDDM: String,
DDMS: String,
DDJDWZ: String,
WFDZ: String,
WFXW: String,
WFJFS: String,
FKJE: String,
SCZ: String,
BZZ: String,
ZNJ: String,
ZQMJ: String,
JKFS: String,
FXJG: String,
FXJGMC: String,
CLJG: String,
CLJGMC: String,
CFZL: String,
CLSJ: String,
JKBJ: String,
JKRQ: String,
PZBH: String,
JSJQBJ: String,
JLLX: String,
LRR: String,
LRSJ: String,
JBR1: String,
JBR2: String,
SGDJ: String,
CLDXBJ: String,
JDCCLDXBJ: String,
ZDJLBJ: String,
XXLY: String,
XRMS: String,
DKBJ: String,
JMZNJBJ: String,
ZDBJ: String,
JSJG: String,
FSJG: String,
GXSJ: String,
BZ: String,
YWJYW: String,
ZJMC: String,
CCLZRQ: String,
NL: String,
XB: String,
HCBJ: String,
JD: String,
WD: String,
YLZZ1: String,
YLZZ2: String,
YLZZ3: String,
YLZZ4: String,
YLZZ5: String,
YLZZ6: String,
YLZZ7: String,
YLZZ8: String,
CJFS: String,
WFSJ1: String,
WFDD1: String,
LDDM1: String,
DDMS1: String,
JSRXZ: String,
CLYT: String,
XCFW: String,
DZZB: String,
SFZDRY: String,
XYSFZMHM: String,
XYXM: String,
YLZZ11: String,
YLZZ12: String,
YLZZ13: String,
YLZZ14: String,
YLZZ15: String,
YLZZ16: String,
YLZZ17: String,
YLZZ18: String)
case class VioViolationLow(wfbh: String,
jdslb: String,
jdsbh: String,
wsjyw: String,
ryfl: String,
jszh: String,
dabh: String,
fzjg: String,
zjcx: String,
dsr: String,
zsxzqh: String,
zsxxdz: String,
dh: String,
lxfs: String,
clfl: String,
hpzl: String,
hphm: String,
jdcsyr: String,
syxz: String,
jtfs: String,
wfsj: String,
xzqh: String,
dllx: String,
glxzdj: String,
wfdd: String,
lddm: String,
ddms: String,
ddjdwz: String,
wfdz: String,
wfxw: String,
wfjfs: String,
fkje: String,
scz: String,
bzz: String,
znj: String,
zqmj: String,
jkfs: String,
fxjg: String,
fxjgmc: String,
cljg: String,
cljgmc: String,
cfzl: String,
clsj: String,
jkbj: String,
jkrq: String,
pzbh: String,
jsjqbj: String,
jllx: String,
lrr: String,
lrsj: String,
jbr1: String,
jbr2: String,
sgdj: String,
cldxbj: String,
jdccldxbj: String,
zdjlbj: String,
xxly: String,
xrms: String,
dkbj: String,
jmznjbj: String,
zdbj: String,
jsjg: String,
fsjg: String,
gxsj: String,
bz: String,
ywjyw: String,
zjmc: String,
cclzrq: String,
nl: String,
xb: String,
hcbj: String,
jd: String,
wd: String,
ylzz1: String,
ylzz2: String,
ylzz3: String,
ylzz4: String,
ylzz5: String,
ylzz6: String,
ylzz7: String,
ylzz8: String,
cjfs: String,
wfsj1: String,
wfdd1: String,
lddm1: String,
ddms1: String,
jsrxz: String,
clyt: String,
xcfw: String,
dzzb: String,
sfzdry: String,
xysfzmhm: String,
xyxm: String,
ylzz11: String,
ylzz12: String,
ylzz13: String,
ylzz14: String,
ylzz15: String,
ylzz16: String,
ylzz17: String,
ylzz18: String)
}
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala
......@@ -170,7 +170,7 @@ object WriteToHbase extends Serializable {
r.getAs[String]("RYZT")
)
}).toDF()
datacsvDF.show()
var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment