Skip to content

F
ftp_pic
Commit abb033b4 by 李辅翼
Options

v14

parent 0d846999
Showing with 252 additions and 182 deletions
pom.xml
......@@ -65,6 +65,24 @@
<artifactId>ojdbc6</artifactId>
<version>11.2.0.3</version>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<exclusions>
<exclusion>
<artifactId>log4j-over-slf4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid-spring-boot-starter</artifactId>
<version>1.1.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
......@@ -76,16 +94,12 @@
<version>1.16.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.51</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
......@@ -104,8 +118,31 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<!--必须加上下面的配置，在jar包运行的时候找不到oracle的驱动类-->
<configuration>
<includeSystemScope>
true
</includeSystemScope>
</configuration>
</plugin>
</plugins>
<!-- <plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<layout>ZIP</layout>
<executable>true</executable>
<includes>
<include>
<groupId>nothing</groupId>
<artifactId>nothing</artifactId>
</include>
</includes>
</configuration>
</plugin>
</plugins>-->
</build>
</project>
src/main/java/com/hikcreate/FtpPicApplication.java
package com.hikcreate;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
@EnableScheduling
@MapperScan("com.hikcreate.dao")
public class FtpPicApplication {
public static void main(String[] args) {
......
src/main/java/com/hikcreate/controller/PicController.java
......@@ -107,8 +107,10 @@ public class PicController {
* @param end
*/
@GetMapping("/increDrvPic")
public void increDrvPic(@RequestParam("start") String start, @RequestParam("end") String end) {
@ResponseBody
public ResponseEntity increDrvPic(@RequestParam("start") String start, @RequestParam("end") String end) {
drvPhoto.increDrvPic(start, end);
return new ResponseEntity("complete", HttpStatus.OK);
}
/**
......@@ -136,18 +138,6 @@ public class PicController {
}
@GetMapping("/drvPhotoHis")
public void getHisDrvPhoto(@RequestParam("time") String time, HttpServletResponse response) {
try {
boolean result = drvPhoto.getHisDrvPhoto(time);
JSONObject jsonObject = new JSONObject();
jsonObject.put("result", result);
response.setContentType("application/json;charset=UTF-8");
response.getWriter().write(jsonObject.toJSONString());
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* app端通过此接口获取违法图片
......
src/main/java/com/hikcreate/dao/DriverPhotoMapper.java 0 → 100644
package com.hikcreate.dao;
import com.hikcreate.entity.DriverPhoto;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
public interface DriverPhotoMapper {
List<DriverPhoto> getIncrePhoto();
List<DriverPhoto> getIncrePhotoBySfzmhm(String sfzmhm);
List<DriverPhoto> getIncrePhotoStage(String start, String end);
}
src/main/java/com/hikcreate/drv_photo_pic/DrvPhoto.java
......@@ -11,7 +11,6 @@ import org.springframework.stereotype.Service;
public interface DrvPhoto {
boolean getHisDrvPhoto(String time);
void getIncrementDrvPhoto();
......
src/main/java/com/hikcreate/drv_photo_pic/impl/DrvPhotoImpl.java
package com.hikcreate.drv_photo_pic.impl;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.JsonObject;
import com.github.tobato.fastdfs.domain.fdfs.StorePath;
import com.github.tobato.fastdfs.service.FastFileStorageClient;
import com.hikcreate.dao.DriverPhotoMapper;
import com.hikcreate.drv_photo_pic.DrvPhoto;
import com.hikcreate.entity.DriverPhoto;
import com.hikcreate.entity.PicResult;
import com.hikcreate.service.fdfs.service.FileService;
import com.hikcreate.utils.DateUtil;
import oracle.sql.BLOB;
import org.apache.hadoop.conf.Configuration;
import com.hikcreate.utils.HbaseUtils;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
......@@ -17,9 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.sql.*;
import java.sql.Connection;
import java.util.List;
import java.util.zip.CRC32;
......@@ -45,23 +48,13 @@ public class DrvPhotoImpl implements DrvPhoto {
private FileService fileService;
@Autowired
private SqlHelp sqlHelp;
@Autowired
private HbaseUtils hbaseUtils;
@Autowired
private DriverPhotoMapper driverPhotoMapper;
@Autowired
private FastFileStorageClient storageClient;
private static String startTime;
private static String endTime;
private static String sfzmhm;
private static BLOB zp;
private static String gxsj;
private static String xzqh;
private static String xh;
private static String flag;
private static String rksj;
private static String fzjg;
private static Configuration hbaseConf;
private static org.apache.hadoop.hbase.client.Connection hbaseConn;
private static TableName driverPhotoTablename;
private static Table driverPhototable;
@Value("${hbase.zookeeper.property.clientPort}")
private String hbaseAddress;
@Value("${hbase.zookeeper.quorum}")
......@@ -70,81 +63,30 @@ public class DrvPhotoImpl implements DrvPhoto {
private String hbaseMaster;
@Value("${hbase.drv_photo.table}")
private String driverPhotoTableStr;
private static String[] colValue;
private static CRC32 crc = new CRC32();
/**
* 同步历史图片，图片同步到time时间
*
* @param time
* @return
*/
public boolean getHisDrvPhoto(String time) {
hbaseConf = HBaseConfiguration.create();
hbaseConf.set("hbase.zookeeper.property.clientPort", hbaseAddress);
hbaseConf.set("hbase.zookeeper.quorum", hbaseZkQuorum);
try {
hbaseConn = ConnectionFactory.createConnection(hbaseConf);
endTime = time;
startTime = sqlHelp.getStartTime("select min(GXSJ) mtime from GYJG.DRV_PHOTO", url, username, password);
//循环获取图片
String sql;
String lastTime;
driverPhotoTablename = TableName.valueOf(driverPhotoTableStr);
driverPhototable = hbaseConn.getTable(driverPhotoTablename);
while (endTime.compareTo(startTime) > 0 || endTime.compareTo(startTime) == 0) {
lastTime = DateUtil.formatDate(DateUtil.getPastDate(DateUtil.parseDate(endTime), roundDay));
Connection connection = DriverManager.getConnection(url, username, password);
sql = "select * from GYJG.DRV_PHOTO where to_char(GXSJ,'yyyy-mm-dd')>='" + lastTime + "' and to_char(GXSJ,'yyyy-mm-dd')<'"
+ endTime + "' and ZP is not null and fzjg is not null and sfzmhm is not null ";
PreparedStatement pstm = connection.prepareStatement(sql);
ResultSet resultSet = pstm.executeQuery();
while (resultSet.next()) {
saveToHbase(resultSet, fileService);
}
endTime = lastTime;
resultSet.close();
pstm.close();
connection.close();
Thread.sleep(1000);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
closeTable();
}
return true;
}
/**
* 每天增量同步
*/
@Override
public void getIncrementDrvPhoto() {
hbaseConf = HBaseConfiguration.create();
hbaseConf.set("hbase.zookeeper.property.clientPort", hbaseAddress);
hbaseConf.set("hbase.zookeeper.quorum", hbaseZkQuorum);
org.apache.hadoop.hbase.client.Connection hbseCon = null;
Table driverPhototable=null;
try {
Class.forName("oracle.jdbc.OracleDriver");
Connection connection = DriverManager.getConnection(url, username, password);
String sql = "select * from GYJG.DRV_PHOTO where GXSJ between sysdate-3 and sysdate and ZP is not null and fzjg is not null and sfzmhm is not null";
PreparedStatement pstm = connection.prepareStatement(sql);
ResultSet resultSet = pstm.executeQuery();
hbaseConn = ConnectionFactory.createConnection(hbaseConf);
driverPhotoTablename = TableName.valueOf(driverPhotoTableStr);
driverPhototable = hbaseConn.getTable(driverPhotoTablename);
while (resultSet.next()) {
saveToHbase(resultSet, fileService);
hbseCon=hbaseUtils.getHbseCon(hbaseAddress, hbaseZkQuorum);
driverPhototable = hbaseUtils.getHbaseTable(hbseCon,driverPhotoTableStr);
List<DriverPhoto> driverPhotos=driverPhotoMapper.getIncrePhoto();
for(DriverPhoto driverPhoto:driverPhotos){
saveToHbase(driverPhoto,driverPhototable);
}
resultSet.close();
pstm.close();
connection.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
closeTable();
hbaseUtils.closeHbaseTable(driverPhototable);
hbaseUtils.clodeHbaseCon(hbseCon);
}
}
......@@ -155,84 +97,106 @@ public class DrvPhotoImpl implements DrvPhoto {
*/
@Override
public void insertDrvPhotoById(String sfzmhm) {
hbaseConf = HBaseConfiguration.create();
hbaseConf.set("hbase.zookeeper.property.clientPort", hbaseAddress);
hbaseConf.set("hbase.zookeeper.quorum", hbaseZkQuorum);
org.apache.hadoop.hbase.client.Connection hbseCon = null;
Table driverPhototable=null;
try {
hbaseConn = ConnectionFactory.createConnection(hbaseConf);
driverPhotoTablename = TableName.valueOf(driverPhotoTableStr);
driverPhototable = hbaseConn.getTable(driverPhotoTablename);
String sql = "select * from GYJG.DRV_PHOTO where ZP is not null and sfzmhm=? order by gxsj";
Connection connection = DriverManager.getConnection(url, username, password);
PreparedStatement pstm = connection.prepareStatement(sql);
pstm.setString(1, sfzmhm);
ResultSet resultSet = pstm.executeQuery();
while (resultSet.next()) {
saveToHbase(resultSet, fileService);
}
resultSet.close();
pstm.close();
connection.close();
hbseCon=hbaseUtils.getHbseCon(hbaseAddress, hbaseZkQuorum);
driverPhototable = hbaseUtils.getHbaseTable(hbseCon,driverPhotoTableStr);
List<DriverPhoto> driverPhotos=driverPhotoMapper.getIncrePhotoBySfzmhm(sfzmhm);
saveToHbase(driverPhotos.get(0),driverPhototable);
} catch (Exception e) {
e.printStackTrace();
} finally {
closeTable();
hbaseUtils.closeHbaseTable(driverPhototable);
hbaseUtils.clodeHbaseCon(hbseCon);
}
}
/**
* 同步指定时间段的驾驶人头像
* @param start
* @param end
*/
@Override
public void increDrvPic(String start, String end) {
hbaseConf = HBaseConfiguration.create();
hbaseConf.set("hbase.zookeeper.property.clientPort", hbaseAddress);
hbaseConf.set("hbase.zookeeper.quorum", hbaseZkQuorum);
org.apache.hadoop.hbase.client.Connection hbseCon = null;
Table driverPhototable=null;
try {
hbaseConn = ConnectionFactory.createConnection(hbaseConf);
driverPhotoTablename = TableName.valueOf(driverPhotoTableStr);
driverPhototable = hbaseConn.getTable(driverPhotoTablename);
String sql = "select * from GYJG.DRV_PHOTO where ZP is not null and fzjg='贵A' and sfzmhm is not null " +
"and to_char(GXSJ,'yyyy-mm-dd')>='" + start + "' and to_char(GXSJ,'yyyy-mm-dd')<'" + end + "'";
Connection connection = DriverManager.getConnection(url, username, password);
PreparedStatement pstm = connection.prepareStatement(sql);
ResultSet resultSet = pstm.executeQuery();
while (resultSet.next()) {
saveToHbase(resultSet, fileService);
hbseCon=hbaseUtils.getHbseCon(hbaseAddress, hbaseZkQuorum);
driverPhototable = hbaseUtils.getHbaseTable(hbseCon,driverPhotoTableStr);
List<DriverPhoto> driverPhotos=driverPhotoMapper.getIncrePhotoStage(start,end);
for(DriverPhoto driverPhoto:driverPhotos){
saveToHbase(driverPhoto,driverPhototable);
}
resultSet.close();
pstm.close();
connection.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
closeTable();
hbaseUtils.closeHbaseTable(driverPhototable);
hbaseUtils.clodeHbaseCon(hbseCon);
}
}
private synchronized void saveToHbase(ResultSet resultSet, FileService fileService) throws Exception {
paramResultSet(resultSet);
private synchronized void saveToHbase(DriverPhoto driverPhoto, Table driverPhototable) throws Exception {
//判断图片是否已经存入fastdfs
colValue = new String[2];
String rowkey = sfzmhm;
PicResult picResult = sqlHelp.getFlag(rowkey, driverPhototable, colValue, gxsj);
String[] colValue = new String[2];
String rowkey = driverPhoto.getSfzmhm();
PicResult picResult = sqlHelp.getFlag(rowkey, driverPhototable, colValue, driverPhoto.getGxsj());
if (picResult.getI() != 0) {
if (picResult.getI() == 2) {
try {
fileService.deleteFile(picResult.getUrl());
} catch (Exception e) {
logger.info(driverPhoto.getSfzmhm() + "---------------------" + picResult.getUrl());
e.printStackTrace();
} finally {
Delete delete = new Delete(rowkey.getBytes());
driverPhototable.delete(delete);
}
}
Put put = new Put(rowkey.getBytes());
put.addColumn("info".getBytes(), "sfzmhm".getBytes(), sfzmhm.getBytes());
byte[] bytes1 = sqlHelp.blobToByteZp(zp);
String picUrl = fileService.uploadFile(bytes1, sfzmhm + ".jpeg");
put.addColumn("info".getBytes(), "sfzmhm".getBytes(), (sfzmhm == null ? "null" : sfzmhm).getBytes());
put.addColumn("info".getBytes(), "picUrl".getBytes(), picUrl.getBytes());
put.addColumn("info".getBytes(), "gxsj".getBytes(), (gxsj == null ? "null" : gxsj).getBytes());
put.addColumn("info".getBytes(), "xzqh".getBytes(), (xzqh == null ? "null" : xzqh).getBytes());
put.addColumn("info".getBytes(), "flag".getBytes(), (flag == null ? "null" : flag).getBytes());
put.addColumn("info".getBytes(), "xh".getBytes(), (xh == null ? "null" : xh).getBytes());
put.addColumn("info".getBytes(), "rksj".getBytes(), (rksj == null ? "null" : rksj).getBytes());
put.addColumn("info".getBytes(), "fzjg".getBytes(), (fzjg == null ? "null" : fzjg).getBytes());
put.addColumn("info".getBytes(), "sfzmhm".getBytes(), driverPhoto.getSfzmhm().getBytes());
byte[] bytes1 = sqlHelp.blobToBytes(driverPhoto.getZp());
//上传
int count = 0;
String picUrl=uploadFileStor(bytes1,driverPhoto.getSfzmhm());
while (picUrl==null){
picUrl=uploadFileStor(bytes1,driverPhoto.getSfzmhm());
count++;
if(count>10){
logger.info("10fail--------sfzmhm---------"+driverPhoto.getSfzmhm());
return;
}
}
//验证
byte[] bytesDown = fileService.downloadFile("/" + picUrl);
if(bytesDown.length!=bytes1.length) {
logger.info("length not right driver:-----------"+driverPhoto.getSfzmhm()+"----------"+bytesDown.length);
picUrl=uploadFileStor(bytes1,driverPhoto.getSfzmhm());
}
//构造put
put.addColumn("info".getBytes(),"picLen".getBytes(),String.valueOf(bytes1.length/1024).getBytes());
put.addColumn("info".getBytes(), "sfzmhm".getBytes(), (driverPhoto.getSfzmhm() == null ? "null" : driverPhoto.getSfzmhm()).getBytes());
put.addColumn("info".getBytes(), "picUrl".getBytes(), ("/"+picUrl).getBytes());
put.addColumn("info".getBytes(), "gxsj".getBytes(), (driverPhoto.getGxsj() == null ? "null" : driverPhoto.getGxsj()).getBytes());
put.addColumn("info".getBytes(), "xzqh".getBytes(), (driverPhoto.getXzqh() == null ? "null" : driverPhoto.getXzqh()).getBytes());
put.addColumn("info".getBytes(), "flag".getBytes(), (driverPhoto.getFlag() == null ? "null" : driverPhoto.getFlag()).getBytes());
put.addColumn("info".getBytes(), "xh".getBytes(), (driverPhoto.getXh() == null ? "null" : driverPhoto.getXh()).getBytes());
put.addColumn("info".getBytes(), "rksj".getBytes(), (driverPhoto.getRksj() == null ? "null" : driverPhoto.getRksj()).getBytes());
put.addColumn("info".getBytes(), "fzjg".getBytes(), (driverPhoto.getFzjg() == null ? "null" : driverPhoto.getFzjg()).getBytes());
driverPhototable.put(put);
}
}
private String uploadFileStor(byte[] bytes, String sfzmhm) {
try {
StorePath jpeg = storageClient.uploadFile(new ByteArrayInputStream(bytes), bytes.length, "jpeg", null);
return jpeg.getFullPath();
}catch (Exception e) {
logger.info(sfzmhm+"***************************");
return null;
}
}
public static String getPrefixString(long prefixLong) {
if (prefixLong < 10L) {
......@@ -244,33 +208,6 @@ public class DrvPhotoImpl implements DrvPhoto {
}
}
private void closeTable() {
if (driverPhototable != null) {
try {
driverPhototable.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (hbaseConn != null) {
try {
hbaseConn.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private synchronized static void paramResultSet(ResultSet resultSet) throws SQLException {
sfzmhm = resultSet.getString("SFZMHM");
zp = (BLOB) resultSet.getBlob("ZP");
gxsj = resultSet.getString("GXSJ");
xzqh = resultSet.getString("XZQH");
flag = resultSet.getString("FLAG");
xh = resultSet.getString("XH");
rksj = resultSet.getString("RKSJ");
fzjg = resultSet.getString("FZJG");
}
public static void main(String[] args) {
String url = "ftp://vion6:vion6@52.1.113.218:21/193.2.117.66/kk/2019-07-06/18/2019070618541777800022598.jpg";
......
src/main/java/com/hikcreate/drv_photo_pic/impl/VehicleImpl.java
......@@ -299,8 +299,6 @@ public class VehicleImpl implements Vehicle {
logger.info(xh+"***************************");
return null;
}
}
......
src/main/java/com/hikcreate/entity/DriverPhoto.java 0 → 100644
package com.hikcreate.entity;
import lombok.Data;
import java.io.Serializable;
@Data
public class DriverPhoto implements Serializable {
String sfzmhm;
oracle.sql.BLOB zp;
String gxsj;
String xzqh;
String flag;
String xh;
String rksj;
String fzjg;
}
src/main/java/com/hikcreate/utils/HbaseUtils.java 0 → 100644
package com.hikcreate.utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Table;
import org.springframework.stereotype.Component;
import java.io.IOException;
@Component
public class HbaseUtils {
public Connection getHbseCon(String hbaseAddress, String hbaseZkQuorum) throws IOException {
Configuration hbaseConf = HBaseConfiguration.create();
hbaseConf.set("hbase.zookeeper.property.clientPort", hbaseAddress);
hbaseConf.set("hbase.zookeeper.quorum", hbaseZkQuorum);
return ConnectionFactory.createConnection(hbaseConf);
}
public Table getHbaseTable(Connection hbaseConn, String vehflowHbaseStr) throws IOException {
return hbaseConn.getTable(TableName.valueOf(vehflowHbaseStr));
}
public void closeHbaseTable(Table vehFlowTable) {
if (vehFlowTable != null) {
try {
vehFlowTable.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void clodeHbaseCon(Connection hbaseConn) {
if (hbaseConn != null) {
try {
hbaseConn.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
src/main/resources/application.properties
url=jdbc:oracle:thin:@172.16.17.82:1521:gyjg
username=gyjg_zckj
password=zckj2018
spring.datasource.url=jdbc:oracle:thin:@172.16.17.82:1521:gyjg
spring.datasource.username=gyjg_zckj
spring.datasource.password=zckj2018
spring.datasource.driver-class-name=oracle.jdbc.driver.OracleDriver
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
mybatis.mapper-locations=classpath:mapper/*.xml
mybatis.type-aliases-package=com.hikcreate.entity
logging.level.com.hikcreate.dao=debug
roundDay=-5
redis.cluster=172.16.25.23:7000,172.16.25.23:7001,172.16.25.23:7002,172.16.25.24:7003,172.16.25.24:7004,172.16.25.24:7005
redis.server=172.16.25.23:7000,172.16.25.23:7001,172.16.25.23:7002,172.16.25.24:7003,172.16.25.24:7004,172.16.25.24:7005
......
src/main/resources/mapper/DriverPhotoMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.hikcreate.dao.DriverPhotoMapper">
<select id="getIncrePhoto" resultType="DriverPhoto">
select sfzmhm,zp,gxsj,xzqh,flag,xh,rksj,fzjg from GYJG.DRV_PHOTO where GXSJ between sysdate-3 and sysdate and ZP is not null and fzjg is not null and sfzmhm is not null;
</select>
<select id="getIncrePhotoBySfzmhm" resultType="DriverPhoto">
select sfzmhm,zp,gxsj,xzqh,flag,xh,rksj,fzjg from GYJG.DRV_PHOTO where ZP is not null and sfzmhm=#{sfzmhm} order by gxsj desc
</select>
<select id="getIncrePhotoStage" resultType="DriverPhoto">
select sfzmhm,zp,gxsj,xzqh,flag,xh,rksj,fzjg from GYJG.DRV_PHOTO ZP is not null and fzjg='贵A' and sfzmhm is not null and to_char(GXSJ,'yyyy-mm-dd')<![CDATA[ >= ]]>#{start}
and to_char(GXSJ,'yyyy-mm-dd')<![CDATA[ < ]]>#{end}
</select>
</mapper>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment