Skip to content

F
ftp_pic
Commit 17b690fa by 李辅翼
Options

v5

parent 1a74f06d
Showing with 54 additions and 25 deletions
src/main/java/com/hikcreate/drv_photo_pic/impl/DrvPhotoImpl.java
......@@ -290,10 +290,10 @@ public class DrvPhotoImpl implements DrvPhoto {
}
public static void main(String[] args) {
String s = "{\"captureType\":\"1\",\"carColor\":\"11\",\"carLogo\":\"0\",\"carSpeed\":\"38.00\",\"conLevel\":\"?\",\"dataSource\":\"1\",\"departmentCode\":\"\",\"devCode\":\"520100000900015905\",\"directionCode\":\"4\",\"firmCode\":\"02\",\"jyw\":\"-1\",\"kafkaSource\":\"2\",\"locationDesc\":\"朝阳洞路与望城路交叉口\",\"monitorTime1\":\"2019-07-11 13:49:53.000\",\"monitorTime2\":\"?\",\"monitorTime3\":\"?\",\"monitorTime4\":\"?\",\"picFeatures\":\"\",\"picUrl1\":\"ftp://vion9:vion9@52.1.123.218:21/52.3.120.233/kk/2019-07-11/13/2019071113495366000000060.jpg\",\"picUrl2\":\"\",\"picUrl3\":\"\",\"picsNum\":\"1\",\"plateColor\":\"4\",\"plateNum\":\"无牌\",\"plateType\":\"00\",\"roadNumCode\":\"3\",\"watchTime\":\"2019-07-11 13:49:53.000\"}";
JSONObject jsonObject = JSONObject.parseObject(new String(s));
jsonObject.put("type", "oc");
System.out.println(jsonObject.toJSONString());
// String url="http://193.5.103.124:8083/5251522281554187435986/2019/06/07/11/zhbrt193313019kk/11495774795494205.jpg?fid=156008-358BE80027-2E8B5C6-B190C";
String url="ftp://vion6:vion6@52.1.113.218:21/193.2.117.66/kk/2019-07-06/18/2019070618541777800022598.jpg";
String[] arr = url.split("/");
System.out.println(arr[arr.length-1].split("jpg")[0]+"jpg");
}
}
src/main/java/com/hikcreate/drv_photo_pic/impl/VioPicImpl.java
......@@ -50,6 +50,8 @@ public class VioPicImpl implements VioPic {
private String vioTableStr;
@Value("${ftpUrl}")
private String ftpUrl;
@Value("${httpUrl}")
private String httpUrl;
@Autowired
......@@ -84,36 +86,27 @@ public class VioPicImpl implements VioPic {
try {
String JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
Class.forName(JDBC_DRIVER);
// logger.info(hiveUrl + "-----------" + hivePassWord + "--------" + hiveUser);
Connection connection = DriverManager.getConnection(hiveUrl, hiveUser, hivePassWord);
PreparedStatement pstm = connection.prepareStatement(incrementVioPicSql);
pstm.setString(1, endDay);
pstm.setString(2, startDay);
// incrementVioPicSql="SELECT a.ccarnumber hphm,a.clicensetype hpzl,b.wfsj wfsj,a.cpic1path url1,a.cpic2path url2,a.cpic3path url3 from " +
// "(SELECT * from kakou.vio_violation_pic_his_ods WHERE substr(export_time,0,10)<='"+endDay+"' and substr(export_time,0,10)>'"+startDay+"') a " +
// "INNER JOIN (SELECT * from default.vio_surveil_all WHERE clsj='null' or clsj='' or clsj is null) b WHERE a.ccarnumber=b.hphm " +
// "and a.clicensetype=b.hpzl and substr(a.dillegaldate,0,16)=substr(b.wfsj,0,16)";
// PreparedStatement pstm = connection.prepareStatement(incrementVioPicSql);
// logger.info(pstm.toString());
// logger.info(startDay);
// logger.info(endDay);
ResultSet resultSet = pstm.executeQuery();
hbaseConf = HBaseConfiguration.create();
hbaseConf.set("hbase.zookeeper.property.clientPort", hbaseAddress);
hbaseConf.set("hbase.zookeeper.quorum", hbaseZkQuorum);
// hbaseConf.set("hbase.master", hbaseMaster);
hbaseConn = ConnectionFactory.createConnection(hbaseConf);
// logger.info("success connection --------------------------");
vioTableName = TableName.valueOf(vioTableStr);
vioTable = hbaseConn.getTable(vioTableName);
restTemplate = new RestTemplate();
headers = new HttpHeaders();
method = HttpMethod.POST;
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
// logger.info("dgewhdjdddddddddddddd-------------");
long count=0L;
loop1:
while (resultSet.next()) {
logger.info("数据库查询出东西了------------");
count=count+1;
logger.info("数据库查询出东西了------------"+count);
hphm = resultSet.getString("hphm");
hpzl = resultSet.getString("hpzl");
wfsj = resultSet.getString("wfsj");
......@@ -150,49 +143,78 @@ public class VioPicImpl implements VioPic {
colValue[5] = value;
}
}
logger.info("查询get结果:" + colValue.toString());
// logger.info("查询get结果:" + colValue.toString());
//插入数据操作
//url1
if (url1 != null && !"".equals(url1)) {
//判断之前是否已经插入过图片了，防止多次插入
if (url1.equals(colValue[0]) || url1.equals(colValue[3])) {
logger.info("url1已经插入过了");
continue loop1;
}
if (colValue[0] == null || "".equals(colValue[0])) {
//url1插入ftpurl1
if (url1.startsWith("ftp")) {
//ftp协议
putData("ftpUrl1", "fastDfsUrl1", url1, ftpUrl, fileService);
} else if (url1.startsWith("http")) {
//http协议
putData("ftpUrl1", "fastDfsUrl1", url1, httpUrl, fileService);
}
} else {
if (colValue[3] == null || "".equals(colValue[3])) {
//url1插入ftpurl4
if (url1.startsWith("ftp")) {
putData("ftpUrl4", "fastDfsUrl4", url1, ftpUrl, fileService);
} else if (url1.startsWith("http")) {
putData("ftpUrl4", "fastDfsUrl4", url1, httpUrl, fileService);
}
}
}
}
if (url2 != null && !"".equals(url2)) {
if (url2.equals(colValue[1]) || url2.equals(colValue[4])) {
logger.info("url2已经插入过了");
continue loop1;
}
if (colValue[1] == null || "".equals(colValue[1])) {
//url1插入ftpurl1
if (url2.startsWith("ftp")) {
putData("ftpUrl2", "fastDfsUrl2", url2, ftpUrl, fileService);
} else if (url2.startsWith("http")) {
putData("ftpUrl2", "fastDfsUrl2", url2, httpUrl, fileService);
}
} else {
if (colValue[4] == null || "".equals(colValue[4])) {
//url1插入ftpurl4
if (url2.startsWith("ftp")) {
putData("ftpUrl5", "fastDfsUrl5", url2, ftpUrl, fileService);
} else if (url2.startsWith("http")) {
putData("ftpUrl5", "fastDfsUrl5", url2, httpUrl, fileService);
}
}
}
}
if (url3 != null && !"".equals(url3)) {
if (url3.equals(colValue[2]) || url3.equals(colValue[5])) {
logger.info("url3已经插入过了");
continue loop1;
}
if (colValue[2] == null || "".equals(colValue[2])) {
//url1插入ftpurl1
if (url3.startsWith("ftp")) {
putData("ftpUrl3", "fastDfsUrl3", url3, ftpUrl, fileService);
} else if (url3.startsWith("http")) {
putData("ftpUrl3", "fastDfsUrl3", url3, httpUrl, fileService);
}
} else {
if (colValue[5] == null || "".equals(colValue[5])) {
//url1插入ftpurl4
if (url3.startsWith("ftp")) {
putData("ftpUrl6", "fastDfsUrl6", url3, ftpUrl, fileService);
} else if (url3.startsWith("http")) {
putData("ftpUrl6", "fastDfsUrl6", url3, httpUrl, fileService);
}
}
}
}
......@@ -238,14 +260,17 @@ public class VioPicImpl implements VioPic {
Map<String, byte[]> map = resp.getBody().getMap();
if (map != null && map.size() > 0) {
for (Map.Entry<String, byte[]> entry : map.entrySet()) {
if (url.startsWith("ftp") || url.startsWith("http")) {
String[] arr = url.split("/");
// logger.info("数组++++"+arr.length+"======"+url+"*****"+map.size()+"***"+arr[arr.length - 1]);
// logger.info("进入循环"+fileService);
String fastDfsUrl = fileService.uploadFile(entry.getValue(), arr[arr.length - 1]);
logger.info("fastdfs地址" + fastDfsUrl);
// logger.info("_-------------" + url);
String fastDfsUrl = fileService.uploadFile(entry.getValue(), arr[arr.length - 1].split("jpg")[0] + "jpg");
put.addColumn("useful".getBytes(), hbaseFastCol.getBytes(), fastDfsUrl.getBytes());
put.addColumn("useful".getBytes(), hbaseFtpCol.getBytes(), url.getBytes());
isPut = true;
} else {
logger.info("图片其他存储方式:" + url);
}
}
} else {
logger.info("此ftp地址数据为空：==========================" + url);
......
src/main/java/com/hikcreate/schedul/PicSchedule.java
......@@ -23,18 +23,21 @@ public class PicSchedule {
/**
* 每天早上10点同步增量数据
* 每天早上10点同步驾驶人头像增量数据
*/
@Scheduled(cron = "0 10 * * * *")
@Scheduled(cron = "0 0 10 * * *")
public void getIncrementDrvPhoto(){
drvPhoto.getIncrementDrvPhoto();
}
// @Scheduled(cron = "0 10 * * * *")
/**
* 每天早上10点同步违法增量数据
*/
@Scheduled(cron = "0 0 10 * * *")
public void getIncrementVioPic(){
String date = DateUtil.getDate();
vioPic.getIncrementVioPic(date,DateUtil.formatDate(DateUtil.getPastDate(new Date(),-pastDay)));
vioPic.getIncrementVioPic(DateUtil.formatDate(DateUtil.getPastDate(new Date(),-pastDay)),date);
}
}
src/main/resources/application.properties
......@@ -5,6 +5,7 @@ roundDay=-5
redis.cluster=172.16.25.23:7000,172.16.25.23:7001,172.16.25.23:7002,172.16.25.24:7003,172.16.25.24:7004,172.16.25.24:7005
redis.server=172.16.25.23:7000,172.16.25.23:7001,172.16.25.23:7002,172.16.25.24:7003,172.16.25.24:7004,172.16.25.24:7005
ftpUrl=http://193.5.103.5:80/ftp/testFtpUtil
httpUrl=http://193.5.103.5:80/ftp/testHttpUtil
#hive
hive.url=jdbc:hive2://172.16.25.25:10000/kakou
hive.user=hdfs
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment