Skip to content

D
datatoHbaseAdEs
Commit 0134edf0 by wangkai
Options

斑马信用演示环境数据入库代码

parents
Showing with 2533 additions and 0 deletions
hkys_gy/.idea/compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="hkys_gy" />
</profile>
</annotationProcessing>
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/hydra.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="HydraSettings">
<option name="hydraStorePath" value="G:\sparkCore\hkys_gy\.hydra\idea" />
<option name="noOfCores" value="2" />
<option name="projectRoot" value="G:\sparkCore\hkys_gy" />
<option name="sourcePartitioner" value="auto" />
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/misc.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/scala_compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaCompilerConfiguration">
<profile name="Maven 1" modules="hkys_gy">
<parameters>
<parameter value="-target:jvm-1.7" />
</parameters>
</profile>
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/uiDesigner.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>
\ No newline at end of file
hkys_gy/.idea/workspace.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="d57f3e96-15ce-4778-9b98-96d25ad8ccc1" name="Default" comment="" />
<ignored path="$PROJECT_DIR$/out/" />
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="DatabaseView">
<option name="SHOW_INTERMEDIATE" value="true" />
<option name="GROUP_DATA_SOURCES" value="true" />
<option name="GROUP_SCHEMA" value="true" />
<option name="GROUP_CONTENTS" value="false" />
<option name="SORT_POSITIONED" value="false" />
<option name="SHOW_EMPTY_GROUPS" value="false" />
<option name="AUTO_SCROLL_FROM_SOURCE" value="false" />
<option name="HIDDEN_KINDS">
<set />
</option>
<expand />
<select />
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="152">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="hkys_gy.iml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/hkys_gy.iml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file leaf-file-name="WriteToHbase.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="179">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DateUtils.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="VehicleToHbase.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#2814#2864#0" />
<element signature="e#6726#11186#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="VioViolationToHbase.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" />
<folding>
<element signature="e#131#193#0" expanded="true" />
<element signature="e#2032#4181#0" />
<element signature="e#4922#8349#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DateFormatUtils.scala" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DataToHbaseUtils.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find />
<find />
<find>ZJCX</find>
</findStrings>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbaseUtils.scala" />
<option value="$PROJECT_DIR$/src/main/scala/DateFormatRes.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormatRes.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormat.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormatUtils.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/DateUtils.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/WriteToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/VioViolationToHbase.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/hikcreate/VehicleToHbase.scala" />
</list>
</option>
</component>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="MavenImportPreferences">
<option name="generalSettings">
<MavenGeneralSettings>
<option name="mavenHome" value="D:/maven/apache-maven-3.5.0" />
</MavenGeneralSettings>
</option>
<option name="importingSettings">
<MavenImportingSettings>
<option name="importAutomatically" value="true" />
</MavenImportingSettings>
</option>
</component>
<component name="MavenProjectNavigator">
<treeState>
<expand>
<path>
<item name="" type="16c1761:MavenProjectsStructure$RootNode" />
<item name="hkys_gy" type="9519ce18:MavenProjectsStructure$ProjectNode" />
</path>
<path>
<item name="" type="16c1761:MavenProjectsStructure$RootNode" />
<item name="hkys_gy" type="9519ce18:MavenProjectsStructure$ProjectNode" />
<item name="Lifecycle" type="58874e2:MavenProjectsStructure$LifecycleNode" />
</path>
</expand>
<select />
</treeState>
</component>
<component name="NodePackageJsonFileManager">
<packageJsonPaths />
</component>
<component name="ProjectFrameBounds" extendedState="6">
<option name="x" value="-8" />
<option name="y" value="-8" />
<option name="width" value="976" />
<option name="height" value="1056" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
<item name="hikcreate" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hkys_gy" type="b2602c69:ProjectViewProjectNode" />
<item name="hkys_gy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
<item name="hikcreate" type="462c0819:PsiDirectoryNode" />
<item name="Utils" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
<pane id="AndroidView" />
<pane id="PackagesPane" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="Downloaded.Files.Path.Enabled" value="false" />
<property name="Repository.Attach.JavaDocs" value="false" />
<property name="Repository.Attach.Sources" value="false" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="project.structure.last.edited" value="Modules" />
<property name="project.structure.proportion" value="0.15" />
<property name="project.structure.side.proportion" value="0.2" />
<property name="settings.editor.selected.configurable" value="Scala Compiler" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager" selected="Application.VioViolationToHbase">
<configuration name="DateFormat" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DateFormat" />
<module name="hkys_gy" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.hikcreate.Utils.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration name="VehicleToHbase" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DataToHbase.VehicleToHbase" />
<module name="hkys_gy" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.hikcreate.Utils.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration name="VioViolationToHbase" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DataToHbase.VioViolationToHbase" />
<module name="hkys_gy" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.hikcreate.Utils.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration name="WriteToHbase" type="Application" factoryName="Application" temporary="true">
<option name="MAIN_CLASS_NAME" value="com.hikcreate.DataToHbase.WriteToHbase" />
<module name="hkys_gy" />
<option name="PROGRAM_PARAMETERS" value="drivingliecense_ys" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.hikcreate.Utils.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration default="true" type="Application" factoryName="Application">
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
</configuration>
<configuration default="true" type="JUnit" factoryName="JUnit">
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="TEST_OBJECT" value="class" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<patterns />
</configuration>
<configuration default="true" type="TestNG" factoryName="TestNG">
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="SUITE_NAME" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="GROUP_NAME" />
<option name="TEST_OBJECT" value="CLASS" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="OUTPUT_DIRECTORY" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<option name="USE_DEFAULT_REPORTERS" value="false" />
<option name="PROPERTIES_FILE" />
<properties />
<listeners />
</configuration>
<list>
<item itemvalue="Application.WriteToHbase" />
<item itemvalue="Application.VehicleToHbase" />
<item itemvalue="Application.VioViolationToHbase" />
<item itemvalue="Application.DateFormat" />
</list>
<recent_temporary>
<list>
<item itemvalue="Application.VioViolationToHbase" />
<item itemvalue="Application.VehicleToHbase" />
<item itemvalue="Application.WriteToHbase" />
<item itemvalue="Application.DateFormat" />
</list>
</recent_temporary>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="d57f3e96-15ce-4778-9b98-96d25ad8ccc1" name="Default" comment="" />
<created>1632388856194</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1632388856194</updated>
<workItem from="1632388859476" duration="3852000" />
<workItem from="1632393132962" duration="82033000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="85885000" />
</component>
<component name="ToolWindowManager">
<frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
<editor active="true" />
<layout>
<window_info anchor="right" id="Palette" order="3" />
<window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.50053304" side_tool="true" weight="0.32897604" />
<window_info anchor="right" id="Maven Projects" order="3" weight="0.7969083" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info id="Capture Tool" order="2" />
<window_info id="Designer" order="2" />
<window_info anchor="right" id="Database" order="3" weight="0.32995737" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Messages" order="7" sideWeight="0.49626866" weight="0.32897604" />
<window_info anchor="right" id="Palette&#9;" order="3" />
<window_info id="Image Layers" order="2" />
<window_info anchor="bottom" id="Java Enterprise" order="7" sideWeight="0.49946696" weight="0.32897604" />
<window_info anchor="right" id="Capture Analysis" order="3" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Run" order="2" sideWeight="0.7318763" weight="0.32897604" />
<window_info anchor="bottom" id="Terminal" order="7" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.17643923" />
<window_info anchor="right" id="Bean Validation" order="3" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="right" id="Theme Preview" order="3" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<option name="time" value="9" />
</breakpoint-manager>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3496">
<caret line="152" column="20" lean-forward="true" selection-start-line="152" selection-start-column="20" selection-end-line="152" selection-end-column="20" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="230">
<caret line="13" column="37" lean-forward="true" selection-start-line="14" selection-start-column="53" selection-end-line="14" selection-end-column="72" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/hkys_gy.iml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hbase/hbase-common/1.1.3/hbase-common-1.1.3.jar!/org/apache/hadoop/hbase/util/Bytes.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="196">
<caret line="224" selection-start-line="224" selection-end-line="224" />
<folding>
<element signature="e#7651#7652#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.11/2.1.0/spark-core_2.11-2.1.0.jar!/org/apache/spark/scheduler/ResultTask.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="6" selection-start-line="6" selection-end-line="6" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.11/2.1.0/spark-core_2.11-2.1.0.jar!/org/apache/spark/rdd/MapPartitionsRDD.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="161">
<caret line="7" selection-start-line="7" selection-end-line="7" />
</state>
</provider>
</entry>
<entry file="jar://D:/java/src.zip!/java/lang/Thread.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="265">
<caret line="747" selection-start-line="747" selection-end-line="747" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-sql_2.11/2.1.0/spark-sql_2.11-2.1.0.jar!/org/apache/spark/sql/execution/DeserializeToObjectExec.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="184">
<caret line="8" selection-start-line="8" selection-end-line="8" />
</state>
</provider>
</entry>
<entry file="jar://D:/java/src.zip!/java/util/ArrayList.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1391">
<caret line="432" selection-start-line="432" selection-end-line="432" />
</state>
</provider>
</entry>
<entry file="jar://D:/scala-2.11.8/scala-2.11.8/lib/scala-library.jar!/scala/App.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="23">
<caret line="1" column="6" selection-start-line="1" selection-start-column="6" selection-end-line="1" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/target/classes.timestamp">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/target/maven-archiver/pom.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="3" column="21" selection-start-line="3" selection-start-column="21" selection-end-line="3" selection-end-column="21" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="152">
<caret line="137" column="40" selection-start-line="137" selection-start-column="40" selection-end-line="137" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormatRes.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="4" column="33" lean-forward="true" selection-start-line="4" selection-start-column="33" selection-end-line="4" selection-end-column="33" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DateFormat.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="161">
<caret line="7" column="29" lean-forward="true" selection-start-line="7" selection-start-column="29" selection-end-line="7" selection-end-column="29" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188">
<caret line="18" column="7" selection-start-line="18" selection-start-column="7" selection-end-line="18" selection-end-column="7" />
<folding>
<element signature="e#131#193#0" expanded="true" />
<element signature="e#2032#4181#0" />
<element signature="e#4922#8349#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="179">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#1847#3758#0" />
<element signature="e#4829#6895#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="188">
<caret line="11" column="7" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding>
<element signature="e#35#97#0" expanded="true" />
<element signature="e#2814#2864#0" />
<element signature="e#6726#11186#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="8" column="6" selection-start-line="8" selection-start-column="6" selection-end-line="8" selection-end-column="6" />
<folding>
<element signature="e#29#78#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="4" column="6" selection-start-line="4" selection-start-column="6" selection-end-line="4" selection-end-column="6" />
<folding>
<element signature="e#29#62#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
<caret line="2" column="6" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
<state key="ArtifactsStructureConfigurable.UI">
<settings>
<artifact-editor />
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="FacetStructureConfigurable.UI">
<settings>
<last-edited>No facets are configured</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="GlobalLibrariesConfigurable.UI">
<settings>
<last-edited>scala-sdk-2.11.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="JdkListConfigurable.UI">
<settings>
<last-edited>1.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ModuleStructureConfigurable.UI">
<settings>
<last-edited>hkys_gy</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ProjectLibrariesConfigurable.UI">
<settings>
<last-edited>Maven: asm:asm:3.1</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>
\ No newline at end of file
hkys_gy/hkys_gy.iml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />
\ No newline at end of file
hkys_gy/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.hikcreate</groupId>
<artifactId>hkys_gy</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<hbase.version>1.1.0-cdh5.15.1</hbase.version>
<spark.version>2.3.0</spark.version>
<scala.version>2.11</scala.version>
<fastjson.version>1.2.38</fastjson.version>
<mysqlDriver.version>5.1.47</mysqlDriver.version>
<odbc.version>12.1.0.1-atlassian-hosted</odbc.version>
<impala.driver.version>2.6.4</impala.driver.version>
<kudu.version>1.10.0</kudu.version>
<netty.version>4.4.42</netty.version>
<kafka.version>2.1.1</kafka.version>
<!-- <kafka.version>0.10.0.1</kafka.version>-->
<!--<hbase.version>1.2.5</hbase.version>-->
<hadoop.version>2.7.3</hadoop.version>
<slf4j.version>1.7.1</slf4j.version>
<elsticsearch.version>6.5.4</elsticsearch.version>
</properties>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<!-- 引入spark相关的jar包 -->
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
<artifactId>joda-time</artifactId>
<groupId>joda-time</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<!-- 引入fastjson -->
<!--<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${fastjson.version}</version>
</dependency>-->
<!-- 引入es与spark的集成 -->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-spark-20_2.11</artifactId>
<version>6.5.4</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.5.4</version>
</dependency>
<!--引入hbase -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.1.1</version>
</dependency>
<!-- <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.0.42.Final</version>
</dependency>-->
<dependency>
<groupId>fakepath</groupId>
<artifactId>hbase-spark</artifactId>
<version>1.0.0</version>
<scope>system</scope>
<systemPath>D:/maven/apache-maven-3.5.0/lib/hbase-spark-1.0.0.jar</systemPath>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<args>
<arg>-target:jvm-1.7</arg>
</args>
<jvmArgs>
<jvmArg>-Xss6000K</jvmArg>
</jvmArgs>
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.18.1</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/VehicleToHbase.scala 0 → 100644
package com.hikcreate.DataToHbase
import com.hikcreate.Utils.{DataToHbaseUtils, DateFormatUtils}
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.sql.SparkSession
import org.apache.spark.{SparkConf, SparkContext}
import org.elasticsearch.spark.sql.EsSparkSQL
object VehicleToHbase extends Serializable {
def main(args: Array[String]): Unit = {
/*val spark: SparkSession = SparkSession.builder()
.appName("HbasePut")
.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.master("local[*]").getOrCreate()*/
//告知哪些类型需要序列化
val conf = new SparkConf()
.setAppName("toHbase")
.setMaster("local[*]")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.set("es.nodes","10.197.236.194,10.197.236.201,10.197.236.193")
.set("es.port","9200")
.set("es.write.operation","index")
.registerKryoClasses(Array(classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],classOf[DateFormatUtils]))
val sc = new SparkContext(conf)
val spark = SparkSession.builder().config(conf).getOrCreate()
if (args.length < 0) {
println("传递的参数错误")
System.exit(1)
}
//hbase表名
val tablename=args(0)
val job: Job = new DataToHbaseUtils().DataToHbaseConf(tablename)
val dateUtils=new DateFormatUtils
import spark.implicits._
val datacsv = spark.read.format("com.databricks.spark.csv")
.option("header", "true") //在csv第一行有属性"true"，没有就是"false"
.option("timestampFormat","yyyy-MM-dd HH:mm:ss")
.load("/root/vehicle.csv")
//.load("C:\\Users\\Administrator\\Documents\\vehicle.csv")
//datacsv.createOrReplaceTempView("vehicle")
/*val datafmt=spark.sql(
"""
select
|XH,
|HPZL,
|HPHM,
|CLPP1,
|CLXH,
|CLPP2,
|GCJK,
|ZZG,
|ZZCMC,
|CLSBDH,
|FDJH,
|CLLX,
|CSYS,
|SYXZ,
|SFZMHM,
|SFZMMC,
|SYR,
|SYQ,
|cast(date_format(to_timestamp(CCDJRQ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as CCDJRQ,
|cast(date_format(to_timestamp(DJRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as DJRQ,
|cast(date_format(to_timestamp(YXQZ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as YXQZ,
|cast(date_format(to_timestamp(QZBFQZ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as QZBFQZ,
|FZJG,
|GLBM,
|cast(date_format(to_timestamp(FPRQ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as FPRQ,
|cast(date_format(to_timestamp(FZRQ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as FZRQ,
|cast(date_format(to_timestamp(FDJRQ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as FDJRQ,
|cast(date_format(to_timestamp(FHGZRQ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as FHGZRQ,
|cast(date_format(to_timestamp(FHGZRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as BXZZRQ,
|BPCS,
|BZCS,
|BDJCS,
|DJZSBH,
|ZDJZSHS,
|DABH,
|XZQH,
|ZT,
|DYBJ,
|JBR,
|CLLY,
|LSH,
|FDJXH,
|RLZL,
|PL,
|GL,
|ZXXS,
|CWKC,
|CWKK,
|CWKG,
|HXNBCD,
|HXNBKD,
|HXNBGD,
|GBTHPS,
|ZS,
|ZJ,
|QLJ,
|HLJ,
|LTGG,
|LTS,
|ZZL,
|ZBZL,
|HDZZL,
|HDZK,
|ZQYZL,
|QPZK,
|HPZK,
|HBDBQK,
|cast(date_format(to_timestamp(CCRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as CCRQ,
|HDFS,
|LLPZ1,
|PZBH1,
|LLPZ2,
|PZBH2,
|XSDW,
|XSJG,
|XSRQ,
|JKPZ,
|JKPZHM,
|HGZBH,
|NSZM,
|NSZMBH,
|cast(date_format(to_timestamp(GXRQ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as GXRQ,
|XGZL,
|QMBH,
|HMBH,
|BZ,
|JYW,
|ZSXZQH,
|ZSXXDZ,
|YZBM1,
|LXDH,
|ZZZ,
|ZZXZQH,
|ZZXXDZ,
|YZBM2,
|ZDYZT,
|YXH,
|CYRY,
|DPHGZBH,
|SQDM,
|CLYT,
|YTSX,
|DZYX,
|XSZBH,
|SJHM,
|JYHGBZBH,
|DWBH,
|cast(date_format(to_timestamp(SYQSRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as SYQSRQ,
|cast(date_format(to_timestamp(YQJYQZBFQZ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as YQJYQZBFQZ,
|cast(date_format(to_timestamp(YQJYQZ2, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as YQJYQZ2,
|FDJGS,
|SFYZHGN,
|ZZJGLX,
|WXMBC,
|NCDQSY,
|cast(date_format(to_timestamp(HPQYSJ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as HPQYSJ,
|DZBSXLH,
|SFXNY,
|XNYZL,
|QDDJXH,
|QDDJH,
|QDDJGL,
|CNZZZL,
|CNZZXS,
|CNZZZDL,
|CNZZZDY,
|CNZZDTDY,
|HDWJCD,
|CDQDXSLCGK,
|CDQDXSLCDS,
|YWJYW
|from vehicle
""".stripMargin)*/
val datacsvDF = datacsv.map(r => {
Vehicle(
r.getAs[String]("XH"),
r.getAs[String]("HPZL"),
r.getAs[String]("HPHM"),
r.getAs[String]("CLPP1"),
r.getAs[String]("CLXH"),
r.getAs[String]("CLPP2"),
r.getAs[String]("GCJK"),
r.getAs[String]("ZZG"),
r.getAs[String]("ZZCMC"),
r.getAs[String]("CLSBDH"),
r.getAs[String]("FDJH"),
r.getAs[String]("CLLX"),
r.getAs[String]("CSYS"),
r.getAs[String]("SYXZ"),
r.getAs[String]("SFZMHM"),
r.getAs[String]("SFZMMC"),
r.getAs[String]("SYR"),
r.getAs[String]("SYQ"),
dateUtils.dateFormatRes(r.getAs[String]("CCDJRQ")),
dateUtils.dateFormatRes(r.getAs[String]("DJRQ")),
dateUtils.dateFormatRes(r.getAs[String]("YXQZ")),
dateUtils.dateFormatRes(r.getAs[String]("QZBFQZ")),
r.getAs[String]("FZJG"),
r.getAs[String]("GLBM"),
dateUtils.dateFormatRes(r.getAs[String]("FPRQ")),
dateUtils.dateFormatRes(r.getAs[String]("FZRQ")),
dateUtils.dateFormatRes(r.getAs[String]("FDJRQ")),
dateUtils.dateFormatRes(r.getAs[String]("FHGZRQ")),
dateUtils.dateFormatRes(r.getAs[String]("BXZZRQ")),
r.getAs[String]("BPCS"),
r.getAs[String]("BZCS"),
r.getAs[String]("BDJCS"),
r.getAs[String]("DJZSBH"),
r.getAs[String]("ZDJZSHS"),
r.getAs[String]("DABH"),
r.getAs[String]("XZQH"),
r.getAs[String]("ZT"),
r.getAs[String]("DYBJ"),
r.getAs[String]("JBR"),
r.getAs[String]("CLLY"),
r.getAs[String]("LSH"),
r.getAs[String]("FDJXH"),
r.getAs[String]("RLZL"),
r.getAs[String]("PL"),
r.getAs[String]("GL"),
r.getAs[String]("ZXXS"),
r.getAs[String]("CWKC"),
r.getAs[String]("CWKK"),
r.getAs[String]("CWKG"),
r.getAs[String]("HXNBCD"),
r.getAs[String]("HXNBKD"),
r.getAs[String]("HXNBGD"),
r.getAs[String]("GBTHPS"),
r.getAs[String]("ZS"),
r.getAs[String]("ZJ"),
r.getAs[String]("QLJ"),
r.getAs[String]("HLJ"),
r.getAs[String]("LTGG"),
r.getAs[String]("LTS"),
r.getAs[String]("ZZL"),
r.getAs[String]("ZBZL"),
r.getAs[String]("HDZZL"),
r.getAs[String]("HDZK"),
r.getAs[String]("ZQYZL"),
r.getAs[String]("QPZK"),
r.getAs[String]("HPZK"),
r.getAs[String]("HBDBQK"),
dateUtils.dateFormatRes(r.getAs[String]("CCRQ")),
r.getAs[String]("HDFS"),
r.getAs[String]("LLPZ1"),
r.getAs[String]("PZBH1"),
r.getAs[String]("LLPZ2"),
r.getAs[String]("PZBH2"),
r.getAs[String]("XSDW"),
r.getAs[String]("XSJG"),
r.getAs[String]("XSRQ"),
r.getAs[String]("JKPZ"),
r.getAs[String]("JKPZHM"),
r.getAs[String]("HGZBH"),
r.getAs[String]("NSZM"),
r.getAs[String]("NSZMBH"),
dateUtils.dateFormatRes(r.getAs[String]("GXRQ")),
r.getAs[String]("XGZL"),
r.getAs[String]("QMBH"),
r.getAs[String]("HMBH"),
r.getAs[String]("BZ"),
r.getAs[String]("JYW"),
r.getAs[String]("ZSXZQH"),
r.getAs[String]("ZSXXDZ"),
r.getAs[String]("YZBM1"),
r.getAs[String]("LXDH"),
r.getAs[String]("ZZZ"),
r.getAs[String]("ZZXZQH"),
r.getAs[String]("ZZXXDZ"),
r.getAs[String]("YZBM2"),
r.getAs[String]("ZDYZT"),
r.getAs[String]("YXH"),
r.getAs[String]("CYRY"),
r.getAs[String]("DPHGZBH"),
r.getAs[String]("SQDM"),
r.getAs[String]("CLYT"),
r.getAs[String]("YTSX"),
r.getAs[String]("DZYX"),
r.getAs[String]("XSZBH"),
r.getAs[String]("SJHM"),
r.getAs[String]("JYHGBZBH"),
r.getAs[String]("DWBH"),
dateUtils.dateFormatRes(r.getAs[String]("SYQSRQ")),
dateUtils.dateFormatRes(r.getAs[String]("YQJYQZBFQZ")),
dateUtils.dateFormatRes(r.getAs[String]("YQJYQZ2")),
r.getAs[String]("FDJGS"),
r.getAs[String]("SFYZHGN"),
r.getAs[String]("ZZJGLX"),
r.getAs[String]("WXMBC"),
r.getAs[String]("NCDQSY"),
dateUtils.dateFormatRes(r.getAs[String]("HPQYSJ")),
r.getAs[String]("DZBSXLH"),
r.getAs[String]("SFXNY"),
r.getAs[String]("XNYZL"),
r.getAs[String]("QDDJXH"),
r.getAs[String]("QDDJH"),
r.getAs[String]("QDDJGL"),
r.getAs[String]("CNZZZL"),
r.getAs[String]("CNZZXS"),
r.getAs[String]("CNZZZDL"),
r.getAs[String]("CNZZZDY"),
r.getAs[String]("CNZZDTDY"),
r.getAs[String]("HDWJCD"),
r.getAs[String]("CDQDXSLCGK"),
r.getAs[String]("CDQDXSLCDS"),
r.getAs[String]("YWJYW")
)
}).toDF()
var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => {
val rowkey = Bytes.toBytes(row.getAs[String]("XH"))
columns.map(
column => {
val colvalue = if (row.getAs[String](column) == null) "null" else row.getAs[String](column)
(rowkey, column, colvalue)
}).toList
})
val rerdd = kvRDD.rdd.map(value => {
val family = Bytes.toBytes("info")
val key = value._1
val col = value._2
val put = new Put(value._1)
if (!col.isEmpty) {
put.addColumn(family, Bytes.toBytes(col), Bytes.toBytes(value._3))
}
(new ImmutableBytesWritable, put)
})
rerdd.take(5).foreach(x => {
println(x)
})
rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
EsSparkSQL.saveToEs(datacsvDF,"vehicle_gy_ys/doc",Map("es.mapping.id"->"XH"))
sc.stop()
}
case class Vehicle(
XH: String,
HPZL: String,
HPHM: String,
CLPP1: String,
CLXH: String,
CLPP2: String,
GCJK: String,
ZZG: String,
ZZCMC: String,
CLSBDH: String,
FDJH: String,
CLLX: String,
CSYS: String,
SYXZ: String,
SFZMHM: String,
SFZMMC: String,
SYR: String,
SYQ: String,
CCDJRQ: String,
DJRQ: String,
YXQZ: String,
QZBFQZ: String,
FZJG: String,
GLBM: String,
FPRQ: String,
FZRQ: String,
FDJRQ: String,
FHGZRQ: String,
BXZZRQ: String,
BPCS: String,
BZCS: String,
BDJCS: String,
DJZSBH: String,
ZDJZSHS: String,
DABH: String,
XZQH: String,
ZT: String,
DYBJ: String,
JBR: String,
CLLY: String,
LSH: String,
FDJXH: String,
RLZL: String,
PL: String,
GL: String,
ZXXS: String,
CWKC: String,
CWKK: String,
CWKG: String,
HXNBCD: String,
HXNBKD: String,
HXNBGD: String,
GBTHPS: String,
ZS: String,
ZJ: String,
QLJ: String,
HLJ: String,
LTGG: String,
LTS: String,
ZZL: String,
ZBZL: String,
HDZZL: String,
HDZK: String,
ZQYZL: String,
QPZK: String,
HPZK: String,
HBDBQK: String,
CCRQ: String,
HDFS: String,
LLPZ1: String,
PZBH1: String,
LLPZ2: String,
PZBH2: String,
XSDW: String,
XSJG: String,
XSRQ: String,
JKPZ: String,
JKPZHM: String,
HGZBH: String,
NSZM: String,
NSZMBH: String,
GXRQ: String,
XGZL: String,
QMBH: String,
HMBH: String,
BZ: String,
JYW: String,
ZSXZQH: String,
ZSXXDZ: String,
YZBM1: String,
LXDH: String,
ZZZ: String,
ZZXZQH: String,
ZZXXDZ: String,
YZBM2: String,
ZDYZT: String,
YXH: String,
CYRY: String,
DPHGZBH: String,
SQDM: String,
CLYT: String,
YTSX: String,
DZYX: String,
XSZBH: String,
SJHM: String,
JYHGBZBH: String,
DWBH: String,
SYQSRQ: String,
YQJYQZBFQZ: String,
YQJYQZ2: String,
FDJGS: String,
SFYZHGN: String,
ZZJGLX: String,
WXMBC: String,
NCDQSY: String,
HPQYSJ: String,
DZBSXLH: String,
SFXNY: String,
XNYZL: String,
QDDJXH: String,
QDDJH: String,
QDDJGL: String,
CNZZZL: String,
CNZZXS: String,
CNZZZDL: String,
CNZZZDY: String,
CNZZDTDY: String,
HDWJCD: String,
CDQDXSLCGK: String,
CDQDXSLCDS: String,
YWJYW: String
)
}
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/VioViolationToHbase.scala 0 → 100644
package com.hikcreate.DataToHbase
/**
* @Author kaiwang
* @Date 2021/9/25 15:45
* @Version 1.0
* 数据入库到hbase
*/
import com.hikcreate.Utils.{DataToHbaseUtils, DateFormatUtils}
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.sql.SparkSession
import org.apache.spark.{SparkConf, SparkContext}
import org.elasticsearch.spark.sql.EsSparkSQL
object VioViolationToHbase extends Serializable {
def main(args: Array[String]): Unit = {
/*val spark: SparkSession = SparkSession.builder()
.appName("HbasePut")
.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.master("local[*]").getOrCreate()*/
//告知哪些类型需要序列化
val conf = new SparkConf()
.setAppName("toHbase")
.setMaster("local[*]")
.set("es.nodes","10.197.236.194,10.197.236.201,10.197.236.193")
.set("es.port","9200")
.set("es.write.operation","index")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.registerKryoClasses(Array(classOf[ImmutableBytesWritable],classOf[DateFormatUtils]))
val sc = new SparkContext(conf)
val spark = SparkSession.builder().config(conf).getOrCreate()
if (args.length < 0) {
println("传递的参数错误")
System.exit(1)
}
//hbase表名
val tablename=args(0)
val job: Job = new DataToHbaseUtils().DataToHbaseConf(tablename)
val dateUtils=new DateFormatUtils
import spark.implicits._
val datacsv = spark.read.format("com.databricks.spark.csv")
.option("header", "true") //在csv第一行有属性"true"，没有就是"false"
.option("timestampFormat","yyyy-MM-dd HH:mm:ss")
.load("/root/VioViolation.csv")
//.load("C:\\Users\\Administrator\\Documents\\VioViolation.csv")
//datacsv.createOrReplaceTempView("VioViolation")
/*val datafmt=spark.sql(
"""
|select
|WFBH,
|JDSLB,
|JDSBH,
|WSJYW,
|RYFL,
|JSZH,
|DABH,
|FZJG,
|ZJCX,
|DSR,
|ZSXZQH,
|ZSXXDZ,
|DH,
|LXFS,
|CLFL,
|HPZL,
|HPHM,
|JDCSYR,
|SYXZ,
|JTFS,
|cast(date_format(to_timestamp(WFSJ,'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as WFSJ,
|XZQH,
|DLLX,
|GLXZDJ,
|WFDD,
|LDDM,
|DDMS,
|DDJDWZ,
|WFDZ,
|WFXW,
|WFJFS,
|FKJE,
|SCZ,
|BZZ,
|ZNJ,
|ZQMJ,
|JKFS,
|FXJG,
|FXJGMC,
|CLJG,
|CLJGMC,
|CFZL,
|cast(date_format(to_timestamp(CLSJ,'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as CLSJ,
|JKBJ,
|cast(date_format(to_timestamp(JKRQ,'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as JKRQ,
|PZBH,
|JSJQBJ,
|JLLX,
|LRR,
|cast(date_format(to_timestamp(LRSJ,'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as LRSJ,
|JBR1,
|JBR2,
|SGDJ,
|CLDXBJ,
|JDCCLDXBJ,
|ZDJLBJ,
|XXLY,
|XRMS,
|DKBJ,
|JMZNJBJ,
|ZDBJ,
|JSJG,
|FSJG,
|cast(date_format(to_timestamp(LRSJ,'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as GXSJ,
|BZ,
|YWJYW,
|ZJMC,
|CCLZRQ,
|NL,
|XB,
|HCBJ,
|JD,
|WD,
|YLZZ1,
|YLZZ2,
|YLZZ3,
|YLZZ4,
|YLZZ5,
|YLZZ6,
|YLZZ7,
|YLZZ8,
|CJFS,
|WFSJ1,
|WFDD1,
|LDDM1,
|DDMS1,
|JSRXZ,
|CLYT,
|XCFW,
|DZZB,
|SFZDRY,
|XYSFZMHM,
|XYXM,
|YLZZ11,
|YLZZ12,
|YLZZ13,
|YLZZ14,
|YLZZ15,
|YLZZ16,
|YLZZ17,
|YLZZ18
|from VioViolation
""".stripMargin)*/
val datacsvDF = datacsv.map(r => {
VioViolation(
r.getAs[String]("WFBH"),
r.getAs[String]("JDSLB"),
r.getAs[String]("JDSBH"),
r.getAs[String]("WSJYW"),
r.getAs[String]("RYFL"),
r.getAs[String]("JSZH"),
r.getAs[String]("DABH"),
r.getAs[String]("FZJG"),
r.getAs[String]("ZJCX"),
r.getAs[String]("DSR"),
r.getAs[String]("ZSXZQH"),
r.getAs[String]("ZSXXDZ"),
r.getAs[String]("DH"),
r.getAs[String]("LXFS"),
r.getAs[String]("CLFL"),
r.getAs[String]("HPZL"),
r.getAs[String]("HPHM"),
r.getAs[String]("JDCSYR"),
r.getAs[String]("SYXZ"),
r.getAs[String]("JTFS"),
dateUtils.dateFormatRes(r.getAs[String]("WFSJ")),
r.getAs[String]("XZQH"),
r.getAs[String]("DLLX"),
r.getAs[String]("GLXZDJ"),
r.getAs[String]("WFDD"),
r.getAs[String]("LDDM"),
r.getAs[String]("DDMS"),
r.getAs[String]("DDJDWZ"),
r.getAs[String]("WFDZ"),
r.getAs[String]("WFXW"),
r.getAs[String]("WFJFS"),
r.getAs[String]("FKJE"),
r.getAs[String]("SCZ"),
r.getAs[String]("BZZ"),
r.getAs[String]("ZNJ"),
r.getAs[String]("ZQMJ"),
r.getAs[String]("JKFS"),
r.getAs[String]("FXJG"),
r.getAs[String]("FXJGMC"),
r.getAs[String]("CLJG"),
r.getAs[String]("CLJGMC"),
r.getAs[String]("CFZL"),
dateUtils.dateFormatRes(r.getAs[String]("CLSJ")),
r.getAs[String]("JKBJ"),
dateUtils.dateFormatRes(r.getAs[String]("JKRQ")),
r.getAs[String]("PZBH"),
r.getAs[String]("JSJQBJ"),
r.getAs[String]("JLLX"),
r.getAs[String]("LRR"),
dateUtils.dateFormatRes(r.getAs[String]("LRSJ")),
r.getAs[String]("JBR1"),
r.getAs[String]("JBR2"),
r.getAs[String]("SGDJ"),
r.getAs[String]("CLDXBJ"),
r.getAs[String]("JDCCLDXBJ"),
r.getAs[String]("ZDJLBJ"),
r.getAs[String]("XXLY"),
r.getAs[String]("XRMS"),
r.getAs[String]("DKBJ"),
r.getAs[String]("JMZNJBJ"),
r.getAs[String]("ZDBJ"),
r.getAs[String]("JSJG"),
r.getAs[String]("FSJG"),
dateUtils.dateFormatRes(r.getAs[String]("GXSJ")),
r.getAs[String]("BZ"),
r.getAs[String]("YWJYW"),
r.getAs[String]("ZJMC"),
r.getAs[String]("CCLZRQ"),
r.getAs[String]("NL"),
r.getAs[String]("XB"),
r.getAs[String]("HCBJ"),
r.getAs[String]("JD"),
r.getAs[String]("WD"),
r.getAs[String]("YLZZ1"),
r.getAs[String]("YLZZ2"),
r.getAs[String]("YLZZ3"),
r.getAs[String]("YLZZ4"),
r.getAs[String]("YLZZ5"),
r.getAs[String]("YLZZ6"),
r.getAs[String]("YLZZ7"),
r.getAs[String]("YLZZ8"),
r.getAs[String]("CJFS"),
r.getAs[String]("WFSJ1"),
r.getAs[String]("WFDD1"),
r.getAs[String]("LDDM1"),
r.getAs[String]("DDMS1"),
r.getAs[String]("JSRXZ"),
r.getAs[String]("CLYT"),
r.getAs[String]("XCFW"),
r.getAs[String]("DZZB"),
r.getAs[String]("SFZDRY"),
r.getAs[String]("XYSFZMHM"),
r.getAs[String]("XYXM"),
r.getAs[String]("YLZZ11"),
r.getAs[String]("YLZZ12"),
r.getAs[String]("YLZZ13"),
r.getAs[String]("YLZZ14"),
r.getAs[String]("YLZZ15"),
r.getAs[String]("YLZZ16"),
r.getAs[String]("YLZZ17"),
r.getAs[String]("YLZZ18")
)
}).toDF()
var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => {
val rowkey = Bytes.toBytes(row.getAs[String]("WFBH"))
columns.map(
column => {
val colvalue = if (row.getAs[String](column) == null) "null" else row.getAs[String](column)
(rowkey, column, colvalue)
}).toList
})
val rerdd = kvRDD.rdd.map(value => {
val family = Bytes.toBytes("info")
val key = value._1
val col = value._2
val put = new Put(value._1)
put.addColumn(family, Bytes.toBytes(col), Bytes.toBytes(value._3))
(new ImmutableBytesWritable, put)
})
rerdd.take(5).foreach(x => {
println(x)
})
rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
EsSparkSQL.saveToEs(datacsvDF,"violation_ys/doc",Map("es.mapping.id"->"WFBH"))
sc.stop()
}
case class VioViolation(WFBH: String,
JDSLB: String,
JDSBH: String,
WSJYW: String,
RYFL: String,
JSZH: String,
DABH: String,
FZJG: String,
ZJCX: String,
DSR: String,
ZSXZQH: String,
ZSXXDZ: String,
DH: String,
LXFS: String,
CLFL: String,
HPZL: String,
HPHM: String,
JDCSYR: String,
SYXZ: String,
JTFS: String,
WFSJ: String,
XZQH: String,
DLLX: String,
GLXZDJ: String,
WFDD: String,
LDDM: String,
DDMS: String,
DDJDWZ: String,
WFDZ: String,
WFXW: String,
WFJFS: String,
FKJE: String,
SCZ: String,
BZZ: String,
ZNJ: String,
ZQMJ: String,
JKFS: String,
FXJG: String,
FXJGMC: String,
CLJG: String,
CLJGMC: String,
CFZL: String,
CLSJ: String,
JKBJ: String,
JKRQ: String,
PZBH: String,
JSJQBJ: String,
JLLX: String,
LRR: String,
LRSJ: String,
JBR1: String,
JBR2: String,
SGDJ: String,
CLDXBJ: String,
JDCCLDXBJ: String,
ZDJLBJ: String,
XXLY: String,
XRMS: String,
DKBJ: String,
JMZNJBJ: String,
ZDBJ: String,
JSJG: String,
FSJG: String,
GXSJ: String,
BZ: String,
YWJYW: String,
ZJMC: String,
CCLZRQ: String,
NL: String,
XB: String,
HCBJ: String,
JD: String,
WD: String,
YLZZ1: String,
YLZZ2: String,
YLZZ3: String,
YLZZ4: String,
YLZZ5: String,
YLZZ6: String,
YLZZ7: String,
YLZZ8: String,
CJFS: String,
WFSJ1: String,
WFDD1: String,
LDDM1: String,
DDMS1: String,
JSRXZ: String,
CLYT: String,
XCFW: String,
DZZB: String,
SFZDRY: String,
XYSFZMHM: String,
XYXM: String,
YLZZ11: String,
YLZZ12: String,
YLZZ13: String,
YLZZ14: String,
YLZZ15: String,
YLZZ16: String,
YLZZ17: String,
YLZZ18: String)
}
hkys_gy/src/main/scala/com/hikcreate/DataToHbase/WriteToHbase.scala 0 → 100644
package com.hikcreate.DataToHbase
import com.hikcreate.Utils.{DataToHbaseUtils, DateFormatUtils}
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapreduce._
import org.apache.spark.sql.SparkSession
import org.apache.spark.{SparkConf, SparkContext}
object WriteToHbase extends Serializable {
def main(args: Array[String]): Unit = {
//告知哪些类型需要序列化
val conf = new SparkConf()
.setAppName("toHbase")
.setMaster("local[*]")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.registerKryoClasses(Array(classOf[ImmutableBytesWritable],classOf[DateFormatUtils]))
val sc = new SparkContext(conf)
val spark = SparkSession.builder().config(conf).getOrCreate()
if (args.length < 0) {
println("传递的参数错误")
System.exit(1)
}
//hbase表名
val tablename=args(0)
val job:Job =new DataToHbaseUtils().DataToHbaseConf(tablename)
val dateUtils=new DateFormatUtils
import spark.implicits._
val datacsv = spark.read.format("com.databricks.spark.csv")
.option("header", "true") //在csv第一行有属性"true"，没有就是"false"
//.load("C:\\Users\\Administrator\\Documents\\drivinglicense.csv")
.load("/root/drivinglicense.csv")
//datacsv.createOrReplaceTempView("drivinglicense")
/*val datafmt=spark.sql(
"""
|select
|DABH,
|SFZMHM,
|ZJCX,
|YZJCX,
|cast(date_format(to_timestamp(QFRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as QFRQ,
|cast(date_format(to_timestamp(SYRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as SYRQ,
|cast(date_format(to_timestamp(CCLZRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as CCLZRQ,
|CCFZJG,
|JZQX,
|cast(date_format(to_timestamp(YXQS, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as YXQS,
|cast(date_format(to_timestamp(YXQZ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as YXQZ,
|LJJF,
|CFRQ,
|XXTZRQ,
|BZCS,
|ZT,
|LY,
|JXMC,
|JLY,
|XZQH,
|XZQJ,
|cast(date_format(to_timestamp(FZRQ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as FZRQ,
|JBR,
|GLBM,
|FZJG,
|cast(date_format(to_timestamp(GXSJ, 'yyyy/MM/dd HH:mm:ss'), 'yyyy-MM-dd HH:mm:ss') as string) as GXSJ,
|LSH,
|XGZL,
|BZ,
|JYW,
|YDABH,
|SQDM,
|ZXBH,
|XH,
|SFZMMC,
|HMCD,
|XM,
|XB,
|cast(date_format(to_timestamp(CSRQ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as CSRQ,
|GJ,
|DJZSXZQH,
|DJZSXXDZ,
|LXZSXZQH,
|LXZSXXDZ,
|LXZSYZBM,
|LXDH,
|SJHM,
|DZYX,
|ZZZM,
|ZZFZJG,
|ZZFZRQ,
|SFBD,
|DWBH,
|cast(date_format(to_timestamp(SYYXQZ, 'yyyy/MM/dd'), 'yyyy-MM-dd') as string) as SYYXQZ,
|XCZG,
|XCZJCX,
|RYZT
|from drivinglicense
""".stripMargin)*/
//datafmt.show()
val datacsvDF = datacsv.map(r => {
Drivinglicense(
r.getAs[String]("DABH"),
r.getAs[String]("SFZMHM"),
r.getAs[String]("ZJCX"),
r.getAs[String]("YZJCX"),
dateUtils.dateFormatRes(r.getAs[String]("QFRQ")),
dateUtils.dateFormatRes(r.getAs[String]("SYRQ")),
dateUtils.dateFormatRes(r.getAs[String]("CCLZRQ")),
r.getAs[String]("CCFZJG"),
r.getAs[String]("JZQX"),
dateUtils.dateFormatRes(r.getAs[String]("YXQS")),
dateUtils.dateFormatRes(r.getAs[String]("YXQZ")),
r.getAs[String]("LJJF"),
dateUtils.dateFormatRes(r.getAs[String]("CFRQ")),
dateUtils.dateFormatRes(r.getAs[String]("XXTZRQ")),
r.getAs[String]("BZCS"),
r.getAs[String]("ZT"),
r.getAs[String]("LY"),
r.getAs[String]("JXMC"),
r.getAs[String]("JLY"),
r.getAs[String]("XZQH"),
r.getAs[String]("XZQJ"),
dateUtils.dateFormatRes(r.getAs[String]("FZRQ")),
r.getAs[String]("JBR"),
r.getAs[String]("GLBM"),
r.getAs[String]("FZJG"),
dateUtils.dateFormatRes(r.getAs[String]("GXSJ")),
r.getAs[String]("LSH"),
r.getAs[String]("XGZL"),
r.getAs[String]("BZ"),
r.getAs[String]("JYW"),
r.getAs[String]("YDABH"),
r.getAs[String]("SQDM"),
r.getAs[String]("ZXBH"),
r.getAs[String]("XH"),
r.getAs[String]("SFZMMC"),
r.getAs[String]("HMCD"),
r.getAs[String]("XM"),
r.getAs[String]("XB"),
dateUtils.dateFormatRes(r.getAs[String]("CSRQ")),
r.getAs[String]("GJ"),
r.getAs[String]("DJZSXZQH"),
r.getAs[String]("DJZSXXDZ"),
r.getAs[String]("LXZSXZQH"),
r.getAs[String]("LXZSXXDZ"),
r.getAs[String]("LXZSYZBM"),
r.getAs[String]("LXDH"),
r.getAs[String]("SJHM"),
r.getAs[String]("DZYX"),
r.getAs[String]("ZZZM"),
r.getAs[String]("ZZFZJG"),
r.getAs[String]("ZZFZRQ"),
r.getAs[String]("SFBD"),
r.getAs[String]("DWBH"),
dateUtils.dateFormatRes(r.getAs[String]("SYYXQZ")),
r.getAs[String]("XCZG"),
r.getAs[String]("XCZJCX"),
r.getAs[String]("RYZT")
)
}).toDF()
datacsvDF.show()
var columns = datacsvDF.columns
val kvRDD = datacsvDF.flatMap(row => {
val rowkey = Bytes.toBytes(row.getAs[String]("SFZMHM"))
columns.map(
column => {
val colvalue = if(row.getAs[String](column) ==null) "null" else row.getAs[String](column)
(rowkey, column, colvalue)
}).toList
})
val rerdd=kvRDD.rdd.map(value => {
val family=Bytes.toBytes("info")
val key=value._1
val col=value._2
val put = new Put(value._1)
if(!col.isEmpty) {
put.addColumn(family, Bytes.toBytes(col),Bytes.toBytes(value._3))
}
(new ImmutableBytesWritable, put)
})
rerdd.take(5).foreach(x=>{println(x)})
rerdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
sc.stop()
}
case class Drivinglicense(DABH: String,
SFZMHM: String,
ZJCX: String,
YZJCX: String,
QFRQ: String,
SYRQ: String,
CCLZRQ: String,
CCFZJG: String,
JZQX: String,
YXQS: String,
YXQZ: String,
LJJF: String,
CFRQ: String,
XXTZRQ: String,
BZCS: String,
ZT: String,
LY: String,
JXMC: String,
JLY: String,
XZQH: String,
XZQJ: String,
FZRQ: String,
JBR: String,
GLBM: String,
FZJG: String,
GXSJ: String,
LSH: String,
XGZL: String,
BZ: String,
JYW: String,
YDABH: String,
SQDM: String,
ZXBH: String,
XH: String,
SFZMMC: String,
HMCD: String,
XM: String,
XB: String,
CSRQ: String,
GJ: String,
DJZSXZQH: String,
DJZSXXDZ: String,
LXZSXZQH: String,
LXZSXXDZ: String,
LXZSYZBM: String,
LXDH: String,
SJHM: String,
DZYX: String,
ZZZM: String,
ZZFZJG: String,
ZZFZRQ: String,
SFBD: String,
DWBH: String,
SYYXQZ: String,
XCZG: String,
XCZJCX: String,
RYZT: String
)
}
\ No newline at end of file
hkys_gy/src/main/scala/com/hikcreate/Utils/DataToHbaseUtils.scala 0 → 100644
package com.hikcreate.Utils
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
import org.apache.hadoop.mapreduce.Job
class DataToHbaseUtils {
def DataToHbaseConf(tablename:String) = {
val hbaseconf = HBaseConfiguration.create()
hbaseconf.set("hbase.zookeeper.quorum", "10.197.236.216,10.197.236.217,10.197.236.218")
hbaseconf.set("hbase.zookeeper.property.clientPort", "2181")
hbaseconf.set("hbase.defaults.for.version.skip", "true")
hbaseconf.set("zookeeper.znode.parent", "/hbase-unsecure")
hbaseconf.set(TableOutputFormat.OUTPUT_TABLE, tablename)
//初始化jobconf，TableOutputFormat必须是org.apache.hadoop.hbase.mapred包下的
val job = Job.getInstance(hbaseconf)
job.setOutputFormatClass(classOf[TableOutputFormat[ImmutableBytesWritable]])
job.setOutputValueClass(classOf[Put])
job.setOutputKeyClass(classOf[ImmutableBytesWritable])
job
}
}
hkys_gy/src/main/scala/com/hikcreate/Utils/DateFormatUtils.scala 0 → 100644
package com.hikcreate.Utils
class DateFormatUtils extends Serializable {
def dateFormatRes(datetime: String) = {
if (datetime == null) {
"null"
} else {
val datemk = new DateUtils(datetime)
val year = datemk.YEAR
val month = if (datemk.MONTH < 10)s"""0${datemk.MONTH}""" else datemk.MONTH
val day = if (datemk.DAY_OF_MONTH < 10)s"""0${datemk.DAY_OF_MONTH}""" else datemk.DAY_OF_MONTH
val hour = if (datemk.HOUR_OF_DAY < 10)s"""0${datemk.HOUR_OF_DAY}""" else datemk.HOUR_OF_DAY
val minute = if (datemk.MINUTE < 10)s"""0${datemk.MINUTE}""" else datemk.MINUTE
val second = if (datemk.SECOND < 10)s"""0${datemk.SECOND}""" else datemk.SECOND
(year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second)
}
}
}
hkys_gy/src/main/scala/com/hikcreate/Utils/DateUtils.scala 0 → 100644
package com.hikcreate.Utils
import java.text.SimpleDateFormat
import java.util.{Calendar, Date}
class DateUtils(datetime: String){
val date: Date =toDate()
private val calendar: Calendar = getCalendar()
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val YEAR: Int = if(calendar == null) -1 else calendar.get(Calendar.YEAR)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val MONTH: Int = if(calendar == null) -1 else calendar.get(Calendar.MONTH) + 1
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val QUARTER: Int = if(calendar == null) -1 else getQuarter()
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val WEEK_OF_YEAR: Int = if(calendar == null) -1 else calendar.get(Calendar.WEEK_OF_YEAR)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val WEEK_OF_MONTH: Int = if(calendar == null) -1 else calendar.get(Calendar.WEEK_OF_MONTH)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val DAY_OF_YEAR: Int = if(calendar == null) -1 else calendar.get(Calendar.DAY_OF_YEAR)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val DAY_OF_MONTH: Int = if(calendar == null) -1 else calendar.get(Calendar.DAY_OF_MONTH)
/**
* 若datetime为"null"，则值为-1，如果day_of_week为1，则值为7，否则为day_of_week-1
*/
val DAY_OF_WEEK: Int = if(calendar == null) -1 else if(calendar.get(Calendar.DAY_OF_WEEK) == 1) 7 else calendar.get(Calendar.DAY_OF_WEEK) -1
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val HOUR_OF_DAY: Int = if(calendar == null) -1 else calendar.get(Calendar.HOUR_OF_DAY)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val MINUTE: Int = if(calendar == null) -1 else calendar.get(Calendar.MINUTE)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val SECOND: Int = if(calendar == null) -1 else calendar.get(Calendar.SECOND)
/**
* 若datetime为"null"，则值为-1，否则为其正常值
*/
val MILLISECOND: Int = if(calendar == null) -1 else calendar.get(Calendar.MILLISECOND)
/**
* 若datetime为"null"，则值为null，否则为其正常值
*/
val FIRST_DAY_OF_MONTH: String = if(calendar == null) null else s"""$YEAR-${if(MONTH < 10) "0" + MONTH else MONTH}-01"""
/**
* 若datetime为"null"，则值为null，否则为其正常值
*/
val FIRST_DAY_OF_YEAR: String = if(calendar == null) null else s"""$YEAR-01-01"""
def toString(format: String): String = {
if(date == null){
null
} else {
val quarter = "0" + QUARTER
val month = if(MONTH < 10) s"""0$MONTH""" else MONTH
val dayOfMonth = if(DAY_OF_MONTH < 10) s"""0$DAY_OF_MONTH""" else DAY_OF_MONTH
val hourOfDay = if(HOUR_OF_DAY < 10) s"""0$HOUR_OF_DAY""" else HOUR_OF_DAY
val minute = if(MINUTE < 10) s"""0$MINUTE""" else MINUTE
val second = if(SECOND < 10) s"""0$SECOND""" else SECOND
val weekOfYear = if(WEEK_OF_YEAR < 10) s"""0$WEEK_OF_YEAR""" else WEEK_OF_YEAR
val weekOfMonth = if(WEEK_OF_MONTH < 10) s"""0$WEEK_OF_MONTH""" else WEEK_OF_MONTH
val formatList = List(
"yyyy",
"QQ",
"MM",
"M",
"dd",
"d",
"HH",
"mm",
"ss",
"yyyyMM",
"yyyy/MM",
"yyyy-MM",
"yyyy年MM月",
"yyyyMMdd",
"yyyy/MM/dd",
"yyyy-MM-dd",
"yyyy年MM月dd日",
"MMdd",
"MM/dd",
"MM-dd",
"MM月dd日",
"yyyyMMdd HH:mm:ss",
"yyyy/MM/dd HH:mm:ss",
"yyyy-MM-dd HH:mm:ss",
"yyyy年MM月dd日 HH时mm分ss秒",
"yyyy年第ww周",
"yyyy年MM月第WW周",
"yyyy年第QQ季度",
"yyyy年QQ季度"
)
if (formatList.contains(format)){
format
.replace("yyyy", s"""$YEAR""")
.replace("MM", s"""$month""")
.replace("M", s"""$MONTH""")
.replace("dd", s"""$dayOfMonth""")
.replace("d", s"""$DAY_OF_MONTH""")
.replace("HH", s"""$hourOfDay""")
.replace("mm", s"""$minute""")
.replace("ss", s"""$second""")
.replace("ww", s"""$weekOfYear""")
.replace("WW", s"""$weekOfMonth""")
.replace("QQ", s"""$quarter""")
} else {
throw new RuntimeException(
s"""
|时间格式错误，当前只支持以下23种格式：
|
|yyyy,
|QQ,
|MM,
|dd,
|HH,
|mm,
|ss,
|yyyyMM,
|yyyy/MM,
|yyyy-MM,
|yyyy年MM月,
|yyyyMMdd,
|yyyy/MM/dd,
|yyyy-MM-dd,
|yyyy年MM月dd日,
|MMdd,
|MM/dd,
|MM-dd,
|MM月dd日,
|yyyyMMdd HH:mm:ss,
|yyyy/MM/dd HH:mm:ss,
|yyyy-MM-dd HH:mm:ss,
|yyyy年MM月dd日 HH时mm分ss秒,
|yyyy年第ww周,
|yyyy年MM月第WW周,
|yyyy年第QQ季度,
|yyyy年QQ季度
|
|您输入的时间格式字符串为：$format
|""".stripMargin)
}
}
}
/**
* 作者：王志豪
* 方法功能：获取calendar对象
* 处理逻辑：判断date是否为空，如果为空则返回null，否则返回date对应的calendar
* @return date对应的calendar
*/
private def getCalendar(): Calendar = {
if(date == null){
null
} else {
val cal = Calendar.getInstance()
cal.setFirstDayOfWeek(Calendar.MONDAY)
cal.setTime(date)
cal
}
}
/**
* 作者：王志豪
* 方法功能：获取date对应的季度
* 处理逻辑：1月到3月为1季度，3月到6月为2季度，7月到9月为3季度，10月到12月为4季度。不在此范围则抛出异常
* @return 返回date对应的季度
*/
private def getQuarter(): Int = {
MONTH match {
case 1 => 1
case 2 => 1
case 3 => 1
case 4 => 2
case 5 => 2
case 6 => 2
case 7 => 3
case 8 => 3
case 9 => 3
case 10 => 4
case 11 => 4
case 12 => 4
case _ => throw new RuntimeException("月份超出正常值外")
}
}
private def toDate(): Date = {
if(datetime == null){
new Date()
} else if ("null".equals(datetime)) {
null
} else {
new SimpleDateFormat(parseFormat(datetime)).parse(datetime)
}
}
/**
* 作者：王志豪
* 方法功能：解析时间格式
* 处理逻辑：1、判断字符串是否为空，如果为空则返回null，不为空则舍去毫秒
* 2、正则匹配时间字符串的格式，并返回格式。
* @param dateString 时间字符串
* @return 时间字符串datestring所对应的时间格式。
*/
private def parseFormat(dateString: String): String = {
val time = if(dateString != null) dateString.split("\\.")(0) else null
if(time == null){
null
} else if(time.matches("^\\d{6}$")){
"yyyyMM"
} else if(time.matches("^\\d{4}/\\d{2}$")){
"yyyy/MM"
} else if(time.matches("^\\d{4}-\\d{2}$")){
"yyyy-MM"
} else if(time.matches("^\\d{4}年\\d{2}月$")){
"yyyy年MM月"
} else if(time.matches("^\\d{8}$")){
"yyyyMMdd"
} else if(time.matches("^\\d{4}/\\d{2}/\\d{2}$")){
"yyyy/MM/dd"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{1}$")){
"yyyy/MM/dd"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{2}$")){
"yyyy/MM/dd"
} else if(time.matches("^\\d{4}/\\d{2}/\\d{1}$")){
"yyyy/MM/dd"
} else if(time.matches("^\\d{4}-\\d{2}-\\d{2}$")){
"yyyy-MM-dd"
} else if(time.matches("^\\d{4}年\\d{2}月\\d{2}日$")){
"yyyy年MM月dd日"
} else if(time.matches("^\\d{8}\\s\\d{2}:\\d{2}:\\d{2}$")){
"yyyyMMdd HH:mm:ss"
} else if(time.matches("^\\d{4}/\\d{2}/\\d{2}\\s\\d{2}:\\d{2}:\\d{2}$")){
"yyyy/MM/dd HH:mm:ss"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{2}\\s\\d{2}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{1}\\s\\d{2}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
} else if(time.matches("^\\d{4}/\\d{1}/\\d{1}\\s\\d{1}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
}else if(time.matches("^\\d{4}/\\d{2}/\\d{2}\\s\\d{1}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
}else if(time.matches("^\\d{4}/\\d{1}/\\d{2}\\s\\d{1}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
}else if(time.matches("^\\d{4}/\\d{2}/\\d{1}\\s\\d{2}:\\d{2}:\\d{2}$")) {
"yyyy/MM/dd HH:mm:ss"
}else if(time.matches("^\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}$")){
"yyyy-MM-dd HH:mm:ss"
} else {
throw new RuntimeException(
s"""
|时间格式错误，当前只支持以下11种格式：
|yyyyMM，
|yyyy-MM，
|yyyy/MM，
|yyyy年MM月，
|yyyyMMdd，
|yyyy-MM-dd，
|yyyy/MM/dd，
|yyyy年MM月dd日，
|yyyyMMdd HH:mm:ss，
|yyyy-MM-dd HH:mm:ss，
|yyyy/MM/dd HH:mm:ss。
|需要解析的时间字符串为：$dateString
|""".stripMargin)
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment