Skip to content

F
ftp_pic
Commit ee8c6c29 by 李辅翼
Options

v14

parent df86ecb6
Showing with 190 additions and 15 deletions
pom.xml
...@@ -120,24 +120,29 @@ ...@@ -120,24 +120,29 @@
<artifactId>fastdfs-client</artifactId> <artifactId>fastdfs-client</artifactId>
<version>1.26.5</version> <version>1.26.5</version>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
</dependencies> </dependencies>
<build> <build>
<plugins> <!-- <plugins>
<plugin> <plugin>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId> <artifactId>spring-boot-maven-plugin</artifactId>
<!--必须加上下面的配置，在jar包运行的时候找不到oracle的驱动类--> &lt;!&ndash;必须加上下面的配置，在jar包运行的时候找不到oracle的驱动类&ndash;&gt;
<configuration> <configuration>
<includeSystemScope> <includeSystemScope>
true true
</includeSystemScope> </includeSystemScope>
</configuration> </configuration>
</plugin> </plugin>
</plugins> </plugins>-->
<!-- <plugins> <plugins>
<plugin> <plugin>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId> <artifactId>spring-boot-maven-plugin</artifactId>
...@@ -152,7 +157,7 @@ ...@@ -152,7 +157,7 @@
</includes> </includes>
</configuration> </configuration>
</plugin> </plugin>
</plugins>--> </plugins>
</build> </build>
......
src/main/java/com/hikcreate/controller/PicController.java
package com.hikcreate.controller; package com.hikcreate.controller;
import com.alibaba.fastjson.JSONObject;
import com.hikcreate.drv_photo_pic.DrvPhoto; import com.hikcreate.drv_photo_pic.DrvPhoto;
import com.hikcreate.drv_photo_pic.VaidData;
import com.hikcreate.drv_photo_pic.Vehicle; import com.hikcreate.drv_photo_pic.Vehicle;
import com.hikcreate.entity.PicByte; import com.hikcreate.entity.PicByte;
import com.hikcreate.service.fdfs.service.FileService; import com.hikcreate.service.fdfs.service.FileService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.LinkedMultiValueMap;
...@@ -29,6 +32,7 @@ import org.springframework.http.*; ...@@ -29,6 +32,7 @@ import org.springframework.http.*;
@RequestMapping("/pic") @RequestMapping("/pic")
public class PicController { public class PicController {
private static Logger logger = LoggerFactory.getLogger(PicController.class);
@Autowired @Autowired
private DrvPhoto drvPhoto; private DrvPhoto drvPhoto;
...@@ -42,6 +46,8 @@ public class PicController { ...@@ -42,6 +46,8 @@ public class PicController {
@Autowired @Autowired
private Vehicle vehicle; private Vehicle vehicle;
@Autowired
private VaidData vaidData;
/** /**
* 插入指定贵阳驾驶人头像 * 插入指定贵阳驾驶人头像
...@@ -113,6 +119,20 @@ public class PicController { ...@@ -113,6 +119,20 @@ public class PicController {
return new ResponseEntity("complete", HttpStatus.OK); return new ResponseEntity("complete", HttpStatus.OK);
} }
/**
* 指定时间段驾驶人头像同步,发证机关为null
*
* @param start
* @param end
*/
@GetMapping("/increDrvPicFzjgNull")
@ResponseBody
public ResponseEntity increDrvPicFzjgNull(@RequestParam("start") String start, @RequestParam("end") String end) {
drvPhoto.increDrvPicFzjgNull(start, end);
return new ResponseEntity("complete", HttpStatus.OK);
}
/** /**
* 根据地址删除图片 * 根据地址删除图片
* @param url * @param url
...@@ -137,6 +157,15 @@ public class PicController { ...@@ -137,6 +157,15 @@ public class PicController {
vehicle.deleteAllVehPic(table,maxrow,minrow); vehicle.deleteAllVehPic(table,maxrow,minrow);
} }
/**
* 校验指定月份的图片长度正常不
* @param table
*/
@GetMapping("/vaidDataLengthByMon")
public void vaidDataLengthByMon(@RequestParam("table") String table,@RequestParam("month") String month){
vaidData.vaidDataLengthByMon(table,month);
}
/** /**
...@@ -146,6 +175,7 @@ public class PicController { ...@@ -146,6 +175,7 @@ public class PicController {
*/ */
@GetMapping("/getVioPic") @GetMapping("/getVioPic")
public synchronized PicByte getVioPicByUrl(@RequestParam("url") String url) { public synchronized PicByte getVioPicByUrl(@RequestParam("url") String url) {
logger.info("app违法数据");
PicByte picByte = new PicByte(); PicByte picByte = new PicByte();
RestTemplate restTemplate = new RestTemplate(); RestTemplate restTemplate = new RestTemplate();
MultiValueMap<String, String> params = new LinkedMultiValueMap<String, String>(); MultiValueMap<String, String> params = new LinkedMultiValueMap<String, String>();
......
src/main/java/com/hikcreate/dao/DriverPhotoMapper.java
...@@ -14,4 +14,6 @@ public interface DriverPhotoMapper { ...@@ -14,4 +14,6 @@ public interface DriverPhotoMapper {
List<DriverPhoto> getIncrePhotoBySfzmhm(String sfzmhm); List<DriverPhoto> getIncrePhotoBySfzmhm(String sfzmhm);
@DBSource(name = "bokang") @DBSource(name = "bokang")
List<DriverPhoto> getIncrePhotoStage(String start, String end); List<DriverPhoto> getIncrePhotoStage(String start, String end);
@DBSource(name = "bokang")
List<DriverPhoto> increDrvPicFzjgNull(String start, String end);
} }
src/main/java/com/hikcreate/drv_photo_pic/DrvPhoto.java
...@@ -18,4 +18,6 @@ public interface DrvPhoto { ...@@ -18,4 +18,6 @@ public interface DrvPhoto {
void increDrvPic(String start, String end); void increDrvPic(String start, String end);
void increDrvPicFzjgNull(String start, String end);
} }
src/main/java/com/hikcreate/drv_photo_pic/VaidData.java 0 → 100644
package com.hikcreate.drv_photo_pic;
public interface VaidData {
void vaidDataLengthByMon(String table, String month);
}
src/main/java/com/hikcreate/drv_photo_pic/impl/SqlHelp.java
...@@ -103,7 +103,7 @@ public class SqlHelp { ...@@ -103,7 +103,7 @@ public class SqlHelp {
} }
} }
public byte[] blobToBytes(Blob blob) { public byte[] blobToBytes(BLOB blob) {
BufferedInputStream is = null; BufferedInputStream is = null;
try { try {
is = new BufferedInputStream(blob.getBinaryStream()); is = new BufferedInputStream(blob.getBinaryStream());
...@@ -121,7 +121,6 @@ public class SqlHelp { ...@@ -121,7 +121,6 @@ public class SqlHelp {
} finally { } finally {
try { try {
is.close(); is.close();
is = null;
} catch (IOException e) { } catch (IOException e) {
return null; return null;
} }
......
src/main/java/com/hikcreate/drv_photo_pic/impl/VaidDataImpl.java 0 → 100644
package com.hikcreate.drv_photo_pic.impl;
import com.github.tobato.fastdfs.service.FastFileStorageClient;
import com.hikcreate.drv_photo_pic.DrvPhoto;
import com.hikcreate.drv_photo_pic.VaidData;
import com.hikcreate.drv_photo_pic.Vehicle;
import com.hikcreate.entity.DriverPhoto;
import com.hikcreate.service.fdfs.service.FileService;
import com.hikcreate.utils.HbaseUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.List;
@Service("vaidDataImpl")
public class VaidDataImpl implements VaidData {
private static Logger logger = LoggerFactory.getLogger(VaidDataImpl.class);
@Autowired
private DrvPhoto drvPhoto;
@Value("${hbase.zookeeper.property.clientPort}")
private String hbaseAddress;
@Value("${hbase.zookeeper.quorum}")
private String hbaseZkQuorum;
@Autowired
private HbaseUtils hbaseUtils;
@Autowired
private FileService fileService;
@Autowired
private Vehicle vehicle;
/**
* 校验指定月份的图片长度正常不
*/
@Override
public void vaidDataLengthByMon(String tableName, String month) {
org.apache.hadoop.hbase.client.Connection hbseCon = null;
Table table = null;
try {
hbseCon = hbaseUtils.getHbseCon(hbaseAddress, hbaseZkQuorum);
table = hbaseUtils.getHbaseTable(hbseCon, tableName);
FilterList filterList = new FilterList();
Scan scan = new Scan();
filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes("info"), Bytes.toBytes("gxsj"), CompareFilter.CompareOp.EQUAL, new SubstringComparator(month)));
filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes("info"), Bytes.toBytes("fzjg"), CompareFilter.CompareOp.EQUAL, new SubstringComparator("贵A")));
// scan.addColumn(Bytes.toBytes("info"), Bytes.toBytes("picUrl"));
// scan.addColumn(Bytes.toBytes("info"), Bytes.toBytes("picLen"));
scan.setFilter(filterList);
// if("pic:drv_photo".equals(tableName)){
// scan.addColumn(Bytes.toBytes("info"), Bytes.toBytes("sfzmhm"));
// }else if("pic:veh_pic_new1".equals(tableName)){
// scan.addColumn(Bytes.toBytes("info"), Bytes.toBytes("xh"));
// }
ResultScanner resultScanner = table.getScanner(scan);
for (Result result : resultScanner) {
List<Cell> cells = result.listCells();
String[] colValue = new String[3];
for (Cell cell : cells) {
String qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
String value = Bytes.toString(CellUtil.cloneValue(cell));
if ("picLen".equals(qualifier)) {
colValue[0] = value;
} else if ("picUrl".equals(qualifier)) {
colValue[1] = value;
} else if("sfzmhm".equals(qualifier) || "xh".equals(qualifier)){
colValue[2] = value;
}
}
try {
byte[] bytes = fileService.downloadFile(colValue[1]);
if (!(bytes.length / 1024+"").equals(colValue[0])) {
logger.info("校验长度不对：++" + colValue[2]+"+++++++++"+colValue[1]+"----------"+colValue[0]+"-----fastdfs长度------"+bytes.length);
// if("pic:drv_photo".equals(tableName)){
// drvPhoto.insertDrvPhotoById(colValue[2]);
// }else if("pic:veh_pic_new1".equals(tableName)){
// vehicle.insertVehPicByXh(colValue[2]);
// }
}
}catch (Exception e){
logger.info("取图片失败：=========="+colValue[2]+"==图片地址=="+colValue[1]);
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
hbaseUtils.closeHbaseTable(table);
hbaseUtils.clodeHbaseCon(hbseCon);
}
}
}
src/main/java/com/hikcreate/drv_photo_pic/impl/VehicleImpl.java
package com.hikcreate.drv_photo_pic.impl; package com.hikcreate.drv_photo_pic.impl;
import com.alibaba.fastjson.JSONObject;
import com.github.tobato.fastdfs.domain.fdfs.StorePath; import com.github.tobato.fastdfs.domain.fdfs.StorePath;
import com.github.tobato.fastdfs.service.FastFileStorageClient; import com.github.tobato.fastdfs.service.FastFileStorageClient;
import com.hikcreate.drv_photo_pic.Vehicle; import com.hikcreate.drv_photo_pic.Vehicle;
...@@ -16,6 +17,7 @@ import org.slf4j.Logger; ...@@ -16,6 +17,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.*; import java.io.*;
...@@ -35,6 +37,10 @@ public class VehicleImpl implements Vehicle { ...@@ -35,6 +37,10 @@ public class VehicleImpl implements Vehicle {
@Autowired @Autowired
private FastFileStorageClient storageClient; private FastFileStorageClient storageClient;
@Autowired
private KafkaTemplate kafkaTemplate;
@Value("pic.topic")
private String picTopic;
@Value("${hbase.veh_pic.table}") @Value("${hbase.veh_pic.table}")
private String vehPicTableStr; private String vehPicTableStr;
@Value("${hbase.zookeeper.property.clientPort}") @Value("${hbase.zookeeper.property.clientPort}")
...@@ -246,6 +252,7 @@ public class VehicleImpl implements Vehicle { ...@@ -246,6 +252,7 @@ public class VehicleImpl implements Vehicle {
colValue = new String[2]; colValue = new String[2];
PicResult picResult = sqlHelp.getFlag(rowkey, vehPicTable, colValue, gxsj); PicResult picResult = sqlHelp.getFlag(rowkey, vehPicTable, colValue, gxsj);
if (picResult.getI() != 0) { if (picResult.getI() != 0) {
// if (picResult.getI() == 1) {
if (picResult.getI() == 2) { if (picResult.getI() == 2) {
try { try {
fileService.deleteFile(picResult.getUrl()); fileService.deleteFile(picResult.getUrl());
...@@ -263,7 +270,6 @@ public class VehicleImpl implements Vehicle { ...@@ -263,7 +270,6 @@ public class VehicleImpl implements Vehicle {
put.addColumn("info".getBytes(), "hpzl".getBytes(), (hpzl == null ? "null" : hpzl).getBytes()); put.addColumn("info".getBytes(), "hpzl".getBytes(), (hpzl == null ? "null" : hpzl).getBytes());
put.addColumn("info".getBytes(), "gxsj".getBytes(), (gxsj == null ? "null" : gxsj).getBytes()); put.addColumn("info".getBytes(), "gxsj".getBytes(), (gxsj == null ? "null" : gxsj).getBytes());
byte[] bytes = sqlHelp.blobToBytes(zp); byte[] bytes = sqlHelp.blobToBytes(zp);
// logger.info(xh+"---------------------"+bytes.length);
put.addColumn("info".getBytes(),"picLen".getBytes(),String.valueOf(bytes.length/1024).getBytes()); put.addColumn("info".getBytes(),"picLen".getBytes(),String.valueOf(bytes.length/1024).getBytes());
//写入fastdfs //写入fastdfs
int count = 0; int count = 0;
...@@ -285,6 +291,14 @@ public class VehicleImpl implements Vehicle { ...@@ -285,6 +291,14 @@ public class VehicleImpl implements Vehicle {
put.addColumn("info".getBytes(), "picUrl".getBytes(), ("/"+picUrl).getBytes()); put.addColumn("info".getBytes(), "picUrl".getBytes(), ("/"+picUrl).getBytes());
vehPicTable.put(put); vehPicTable.put(put);
vehPicTable.close(); vehPicTable.close();
if (hphm!=null && hphm.length()>4){
JSONObject resultJson = new JSONObject();
resultJson.put("type","sfzmhm");
resultJson.put("row",xh);
resultJson.put("info",hphm+"#"+hpzl);
resultJson.put("picUrl","/" + picUrl);
kafkaTemplate.send(picTopic,resultJson.toString());
}
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
......
src/main/java/com/hikcreate/entity/DriverPhoto.java
...@@ -9,7 +9,7 @@ import java.io.Serializable; ...@@ -9,7 +9,7 @@ import java.io.Serializable;
public class DriverPhoto implements Serializable { public class DriverPhoto implements Serializable {
String sfzmhm; String sfzmhm;
oracle.sql.BLOB zp; byte[] zp;
String gxsj; String gxsj;
String xzqh; String xzqh;
String flag; String flag;
......
src/main/java/com/hikcreate/schedul/PicSchedule.java
...@@ -25,7 +25,7 @@ public class PicSchedule { ...@@ -25,7 +25,7 @@ public class PicSchedule {
/** /**
* 每天早上10点同步驾驶人头像增量数据 * 每天早上10点同步驾驶人头像增量数据
*/ */
// @Scheduled(cron = "0 0 10 * * *") // @Scheduled(cron = "0 0 15 * * ?")
public void getIncrementDrvPhoto(){ public void getIncrementDrvPhoto(){
drvPhoto.getIncrementDrvPhoto(); drvPhoto.getIncrementDrvPhoto();
} }
...@@ -35,7 +35,7 @@ public class PicSchedule { ...@@ -35,7 +35,7 @@ public class PicSchedule {
/** /**
* 每天早上9点同步机动车的增量数据 * 每天早上9点同步机动车的增量数据
*/ */
// @Scheduled(cron = "0 0 9 * * *") // @Scheduled(cron = "0 0 15 * * ?")
public void getIncrementVehPic(){ public void getIncrementVehPic(){
vehicle.getIncrementVehPic(); vehicle.getIncrementVehPic();
} }
......
src/main/resources/application.properties
...@@ -13,12 +13,23 @@ spring.datasource.bokang.password=zckj2018 ...@@ -13,12 +13,23 @@ spring.datasource.bokang.password=zckj2018
spring.datasource.bokang.driver-class-name=oracle.jdbc.driver.OracleDriver spring.datasource.bokang.driver-class-name=oracle.jdbc.driver.OracleDriver
url=jdbc:oracle:thin:@172.16.17.82:1521:gyjg
username=gyjg_zckj
password=zckj2018
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
mybatis.mapper-locations=classpath:mapper/*.xml mybatis.mapper-locations=classpath:mapper/*.xml
mybatis.type-aliases-package=com.hikcreate.entity mybatis.type-aliases-package=com.hikcreate.entity
logging.level.com.hikcreate.dao=debug logging.level.com.hikcreate.dao=debug
spring.kafka.producer.bootstrap-servers=172.16.25.23:9092,172.16.25.24:9092,172.16.25.25:9092,172.16.25.26:9092,172.16.25.27:9092
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
pic.topic=picture
roundDay=-5 roundDay=-5
redis.cluster=172.16.25.23:7000,172.16.25.23:7001,172.16.25.23:7002,172.16.25.24:7003,172.16.25.24:7004,172.16.25.24:7005 redis.cluster=172.16.25.23:7000,172.16.25.23:7001,172.16.25.23:7002,172.16.25.24:7003,172.16.25.24:7004,172.16.25.24:7005
redis.server=172.16.25.23:7000,172.16.25.23:7001,172.16.25.23:7002,172.16.25.24:7003,172.16.25.24:7004,172.16.25.24:7005 redis.server=172.16.25.23:7000,172.16.25.23:7001,172.16.25.23:7002,172.16.25.24:7003,172.16.25.24:7004,172.16.25.24:7005
...@@ -38,7 +49,7 @@ fdfs.tracker-list[1] = 172.16.25.26:22122 ...@@ -38,7 +49,7 @@ fdfs.tracker-list[1] = 172.16.25.26:22122
fdfs.pool.max-total = 200 fdfs.pool.max-total = 200
fdfs.pool.max-wait-millis = 150 fdfs.pool.max-wait-millis = 150
#端口 #端口
server.port=8085 server.port=8084
hbase.zookeeper.property.clientPort=2181 hbase.zookeeper.property.clientPort=2181
hbase.zookeeper.quorum=172.16.25.25,172.16.25.28,172.16.25.24,172.16.25.26,172.16.25.27 hbase.zookeeper.quorum=172.16.25.25,172.16.25.28,172.16.25.24,172.16.25.26,172.16.25.27
......
src/main/resources/mapper/DriverPhotoMapper.xml
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.hikcreate.dao.DriverPhotoMapper"> <mapper namespace="com.hikcreate.dao.DriverPhotoMapper">
<select id="getIncrePhoto" resultType="DriverPhoto"> <select id="getIncrePhoto" resultType="DriverPhoto">
select sfzmhm,zp,gxsj,xzqh,flag,xh,rksj,fzjg from GYJG.DRV_PHOTO where GXSJ between sysdate-3 and sysdate and ZP is not null and fzjg is not null and sfzmhm is not null; select sfzmhm,zp,gxsj,xzqh,flag,xh,rksj,fzjg from GYJG.DRV_PHOTO where GXSJ between sysdate-3 and sysdate and ZP is not null and sfzmhm is not null;
</select> </select>
<select id="getIncrePhotoBySfzmhm" resultType="DriverPhoto"> <select id="getIncrePhotoBySfzmhm" resultType="DriverPhoto">
...@@ -11,8 +11,12 @@ ...@@ -11,8 +11,12 @@
<select id="getIncrePhotoStage" resultType="DriverPhoto"> <select id="getIncrePhotoStage" resultType="DriverPhoto">
select sfzmhm,zp,gxsj,xzqh,flag,xh,rksj,fzjg from GYJG.DRV_PHOTO ZP is not null and fzjg='贵A' and sfzmhm is not null and to_char(GXSJ,'yyyy-mm-dd')<![CDATA[ >= ]]>#{start} select sfzmhm,zp,gxsj,xzqh,flag,xh,rksj,fzjg from GYJG.DRV_PHOTO where ZP is not null and fzjg='贵A' and sfzmhm is not null and to_char(GXSJ,'yyyy-mm-dd')<![CDATA[ >= ]]>#{start}
and to_char(GXSJ,'yyyy-mm-dd')<![CDATA[ < ]]>#{end} and to_char(GXSJ,'yyyy-mm-dd')<![CDATA[ < ]]>#{end}
</select> </select>
<select id="increDrvPicFzjgNull" resultType="DriverPhoto">
select sfzmhm,zp,gxsj,xzqh,flag,xh,rksj,fzjg from GYJG.DRV_PHOTO where ZP is not null and fzjg is null and sfzmhm is not null and to_char(GXSJ,'yyyy-mm-dd')<![CDATA[ >= ]]>#{start}
and to_char(GXSJ,'yyyy-mm-dd')<![CDATA[ < ]]>#{end}
</select>
</mapper> </mapper>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment