|
@@ -1,593 +0,0 @@
|
|
|
-package cc.xiaokuihua.analysis.service.impl;
|
|
|
-
|
|
|
-import cc.xiaokuihua.analysis.analysis.DouYInEvil0ctal;
|
|
|
-import cc.xiaokuihua.analysis.analysis.DouYin;
|
|
|
-import cc.xiaokuihua.analysis.analysis.DouYinHub;
|
|
|
-import cc.xiaokuihua.analysis.config.NacosConfig;
|
|
|
-import cc.xiaokuihua.analysis.service.AnalysisService;
|
|
|
-import cc.xiaokuihua.analysis.service.AnalysisTokenService;
|
|
|
-import cc.xiaokuihua.analysis.service.AnalysisUserService;
|
|
|
-import cc.xiaokuihua.entity.analysis.entity.AnalysisLog;
|
|
|
-import cc.xiaokuihua.entity.analysis.entity.AnalysisToken;
|
|
|
-import cc.xiaokuihua.entity.analysis.entity.AnalysisUser;
|
|
|
-import cc.xiaokuihua.entity.analysis.entity.DouYinAnalysis;
|
|
|
-import cc.xiaokuihua.entity.analysis.mapper.AnalysisLogMapper;
|
|
|
-import cc.xiaokuihua.util.image.WebpConvertUtil;
|
|
|
-import cc.xiaokuihua.util.string.StringUtil;
|
|
|
-import cc.xiaokuihua.util.url.UrlUtils;
|
|
|
-import cc.xiaokuihua.util.webdav.WebDavJackRabbitUtil;
|
|
|
-import cc.xiaokuihua.util.webdav.WebDavUpload;
|
|
|
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
|
-import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
|
|
-import lombok.RequiredArgsConstructor;
|
|
|
-import lombok.extern.slf4j.Slf4j;
|
|
|
-import org.apache.commons.io.FileUtils;
|
|
|
-import org.apache.tomcat.util.http.fileupload.IOUtils;
|
|
|
-import org.springframework.beans.factory.annotation.Value;
|
|
|
-import org.springframework.stereotype.Service;
|
|
|
-
|
|
|
-import javax.servlet.http.HttpServletResponse;
|
|
|
-import java.io.*;
|
|
|
-import java.net.HttpURLConnection;
|
|
|
-import java.net.URL;
|
|
|
-import java.net.URLEncoder;
|
|
|
-import java.nio.charset.StandardCharsets;
|
|
|
-import java.nio.file.Files;
|
|
|
-import java.text.SimpleDateFormat;
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.Date;
|
|
|
-import java.util.List;
|
|
|
-import java.util.zip.ZipEntry;
|
|
|
-import java.util.zip.ZipOutputStream;
|
|
|
-
|
|
|
-/**
|
|
|
- * 解析ServiceImpl
|
|
|
- *
|
|
|
- * @author ssk
|
|
|
- * @version 1.0
|
|
|
- * @date 2023/5/29 16:53
|
|
|
- * @date 2023/12/27
|
|
|
- */
|
|
|
-@Slf4j
|
|
|
-@Service
|
|
|
-@RequiredArgsConstructor
|
|
|
-public class AnalysisServiceImpl implements AnalysisService {
|
|
|
-
|
|
|
- /**
|
|
|
- * 动态获取Nacos配置
|
|
|
- */
|
|
|
- private final NacosConfig nacosConfig;
|
|
|
-
|
|
|
- /**
|
|
|
- * 解析记录Mapper
|
|
|
- */
|
|
|
- private final AnalysisLogMapper analysisLogMapper;
|
|
|
-
|
|
|
- /**
|
|
|
- * 抖音用户Service
|
|
|
- */
|
|
|
- private final AnalysisUserService analysisUserService;
|
|
|
-
|
|
|
- /**
|
|
|
- * 抖音用户TokenService
|
|
|
- */
|
|
|
- private final AnalysisTokenService analysisTokenService;
|
|
|
-
|
|
|
- private final DouYinHub douYinHub;
|
|
|
-
|
|
|
- private final DouYin douYin;
|
|
|
-
|
|
|
- private final DouYInEvil0ctal douYInEvil0ctal;
|
|
|
-
|
|
|
- @Value("${filePath}")
|
|
|
- private String filePath;
|
|
|
-
|
|
|
- @Value("${webdav.url}")
|
|
|
- private String webDavUrl;
|
|
|
-
|
|
|
- @Value("${webdav.username}")
|
|
|
- private String webDavUserName;
|
|
|
-
|
|
|
- @Value("${webdav.password}")
|
|
|
- private String webDavPassWord;
|
|
|
-
|
|
|
- /**************************************DouYinWeb**********************************************/
|
|
|
-
|
|
|
- /**
|
|
|
- * 解析用户主页数据
|
|
|
- *
|
|
|
- * @param link 抖音浏览器版主页链接
|
|
|
- */
|
|
|
- @Override
|
|
|
- public List<DouYinAnalysis> getUserHomePageData(String link) {
|
|
|
- return douYin.getUserAllData(link, nacosConfig.getDouyinCookie(), nacosConfig.getUserAgent());
|
|
|
- }
|
|
|
-
|
|
|
- /**************************************DouYinHubWebApi**********************************************/
|
|
|
-
|
|
|
- /**
|
|
|
- * 下载到本地(直接下载)
|
|
|
- *
|
|
|
- * @param response 响应
|
|
|
- * @param link 链接
|
|
|
- */
|
|
|
- @Override
|
|
|
- public void downloadToLocal(HttpServletResponse response, String link) {
|
|
|
- try {
|
|
|
- //解析链接
|
|
|
- String url = UrlUtils.getUrl(link);
|
|
|
- DouYinAnalysis douYinResult = douYInEvil0ctal.getDouYinResult(url);
|
|
|
- if (douYinResult == null) {
|
|
|
- log.error("downloadToLocal: 解析数据为空");
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
- //所有要下载的链接
|
|
|
- List<String> pathList = new ArrayList<>();
|
|
|
-
|
|
|
- //规范文件名
|
|
|
- String title = StringUtil.getString(douYinResult.getDesc());
|
|
|
-
|
|
|
- //解析的视频链接
|
|
|
- if (null != douYinResult.getVideo_list() && !douYinResult.getVideo_list().isEmpty()) {
|
|
|
- pathList.addAll(douYinResult.getVideo_list());
|
|
|
- }
|
|
|
-
|
|
|
- //解析的图片链接
|
|
|
- if (null != douYinResult.getImage_list() && !douYinResult.getImage_list().isEmpty()) {
|
|
|
- pathList.addAll(douYinResult.getImage_list());
|
|
|
- }
|
|
|
-
|
|
|
- //下载多个文件
|
|
|
- if (pathList.size() > 1) {
|
|
|
- FileInputStream fis;
|
|
|
- String downloadFileName;
|
|
|
- // 创建临时zip文件
|
|
|
- String zipFileName = title + nowTime();
|
|
|
- if (zipFileName.length() > 100) {
|
|
|
- zipFileName = title;
|
|
|
- }
|
|
|
- File zip = File.createTempFile(zipFileName, ".zip");
|
|
|
- ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(zip.toPath()));
|
|
|
- int i = 1;
|
|
|
- for (String s : pathList) {
|
|
|
- log.info("downloadFunction: ItemId:{}, 总文件数:{}, 当前个数:{}", douYinResult.getItemId(), pathList.size(), i);
|
|
|
- HttpURLConnection conn = conn(s);
|
|
|
- String fileType = new String(conn.getHeaderField("Content-Type").getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
|
|
|
- fileType = "." + fileType.substring(fileType.lastIndexOf("/") + 1);
|
|
|
- //得到输入流
|
|
|
- InputStream inputStream = conn.getInputStream();
|
|
|
- if (".webp".equalsIgnoreCase(fileType)) {
|
|
|
- inputStream = WebpConvertUtil.webpConvertWithStream(inputStream, "jpg");
|
|
|
- fileType = ".jpg";
|
|
|
- }
|
|
|
- BufferedInputStream in = new BufferedInputStream(inputStream);
|
|
|
- out.putNextEntry(new ZipEntry(nowTime() + fileType));
|
|
|
- int b;
|
|
|
- while ((b = in.read()) != -1) {
|
|
|
- out.write(b);
|
|
|
- }
|
|
|
- in.close();
|
|
|
- i++;
|
|
|
- }
|
|
|
- out.close();
|
|
|
- fis = new FileInputStream(zip);
|
|
|
- //以防中文文件名乱码
|
|
|
- downloadFileName = URLEncoder.encode(zip.getName(), "UTF-8");
|
|
|
- //处理空格转为加号的问题
|
|
|
- downloadFileName = downloadFileName.replaceAll("\\+", "%20");
|
|
|
- response.setHeader("Content-Disposition", "attachment; filename=" + downloadFileName);
|
|
|
- IOUtils.copy(fis, response.getOutputStream());
|
|
|
- response.flushBuffer();
|
|
|
- try {
|
|
|
- fis.close();
|
|
|
- if (zip.delete()) {
|
|
|
- log.info("downloadToLocal: 打包文件已删除");
|
|
|
- }
|
|
|
- } catch (IOException e) {
|
|
|
- log.error("downloadToLocal: ERROR Msg={}&Stack={}", e.getMessage(), e.getStackTrace());
|
|
|
- }
|
|
|
- } else if (pathList.size() == 1) {
|
|
|
- //单个下载
|
|
|
- HttpURLConnection conn = conn(pathList.get(0));
|
|
|
- String fileType = new String(conn.getHeaderField("Content-Type").getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
|
|
|
- fileType = "." + fileType.substring(fileType.lastIndexOf("/") + 1);
|
|
|
- //得到输入流
|
|
|
- InputStream inputStream = conn.getInputStream();
|
|
|
- if (".webp".equalsIgnoreCase(fileType)) {
|
|
|
- inputStream = WebpConvertUtil.webpConvertWithStream(inputStream, "jpg");
|
|
|
- fileType = ".jpg";
|
|
|
- }
|
|
|
- String fileName = "attachment; filename=" + title + nowTime() + fileType;
|
|
|
- if (fileName.length() > 100) {
|
|
|
- fileName = "attachment; filename=" + nowTime() + fileType;
|
|
|
- }
|
|
|
- response.setHeader("Content-Disposition", new String(fileName.getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1));
|
|
|
- IOUtils.copy(inputStream, response.getOutputStream());
|
|
|
- response.flushBuffer();
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("downloadToLocal: ERROR Msg={}&Stack={}", e.getMessage(), e.getStackTrace());
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取单条视频数据
|
|
|
- *
|
|
|
- * @param link 链接
|
|
|
- */
|
|
|
- @Override
|
|
|
- public DouYinAnalysis getOneVideoData(String link) {
|
|
|
- DouYinAnalysis douYinAnalysis = new DouYinAnalysis();
|
|
|
- //解析链接
|
|
|
- String url = UrlUtils.getUrl(link);
|
|
|
- //项目id
|
|
|
- String itemId = douYin.getItemId(link);
|
|
|
- List<AnalysisLog> list;
|
|
|
- if (itemId == null) {
|
|
|
- list = null;
|
|
|
- } else {
|
|
|
- //查询视频是否下载过
|
|
|
- QueryWrapper<AnalysisLog> queryWrapper = new QueryWrapper<>();
|
|
|
- queryWrapper.eq("link", url).or().eq(!itemId.isEmpty(), "item_id", itemId).orderByDesc("created_time");
|
|
|
- list = analysisLogMapper.selectList(queryWrapper);
|
|
|
- }
|
|
|
- if (list != null && !list.isEmpty()) {
|
|
|
- AnalysisLog one = list.get(0);
|
|
|
- //下载过
|
|
|
- douYinAnalysis.setLink("have");
|
|
|
- if (!itemId.isEmpty()) {
|
|
|
- douYinAnalysis.setItemId(itemId);
|
|
|
- AnalysisLog analysisLog = new AnalysisLog();
|
|
|
- analysisLog.setId(one.getId());
|
|
|
- analysisLog.setItemId(itemId);
|
|
|
- analysisLogMapper.updateById(analysisLog);
|
|
|
- }
|
|
|
- } else {
|
|
|
- //解析链接
|
|
|
- douYinAnalysis = douYInEvil0ctal.getDouYinResult(url);
|
|
|
- if (douYinAnalysis == null) {
|
|
|
- log.error("getOneVideoData: 解析数据为空");
|
|
|
- return null;
|
|
|
- }
|
|
|
- }
|
|
|
- return douYinAnalysis;
|
|
|
- }
|
|
|
-
|
|
|
- /**************************************DouYinHubApi**********************************************/
|
|
|
-
|
|
|
- /**
|
|
|
- * 解析单条视频
|
|
|
- *
|
|
|
- * @param link 链接
|
|
|
- * @return DouYinAnalysis
|
|
|
- */
|
|
|
- @Override
|
|
|
- public DouYinAnalysis douyinHubGetOneVideo(String link) {
|
|
|
- String token = getToken();
|
|
|
- if (!"".equals(token)) {
|
|
|
- //解析链接
|
|
|
- String url = UrlUtils.getUrl(link);
|
|
|
- return douYinHub.getOneVideoData(token, url);
|
|
|
- } else {
|
|
|
- DouYinAnalysis douYinAnalysis = new DouYinAnalysis();
|
|
|
- douYinAnalysis.setLink("token");
|
|
|
- return douYinAnalysis;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取用户主页所有
|
|
|
- *
|
|
|
- * @param link 链接
|
|
|
- * @return List<DouYinAnalysis>
|
|
|
- */
|
|
|
- @Override
|
|
|
- public List<DouYinAnalysis> douyinHubGetUserHomePage(String link) {
|
|
|
- List<DouYinAnalysis> resultList = new ArrayList<>();
|
|
|
- String token = getToken();
|
|
|
- if (!"".equals(token)) {
|
|
|
- //解析链接
|
|
|
- String url = UrlUtils.getUrl(link);
|
|
|
- return douYinHub.getUserVideoData(token, url);
|
|
|
- } else {
|
|
|
- DouYinAnalysis douYinAnalysis = new DouYinAnalysis();
|
|
|
- douYinAnalysis.setLink("token");
|
|
|
- resultList.add(douYinAnalysis);
|
|
|
- return resultList;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**************************************Download**********************************************/
|
|
|
-
|
|
|
- /**
|
|
|
- * 下载
|
|
|
- *
|
|
|
- * @param douYinAnalysisList 解析的单条或多条内容
|
|
|
- * @param flag flag:{0:单条,1:用户主页,2:}
|
|
|
- * @param repeatFlag 去重标记, 单条默认去重(消息模块也会验证重复性)
|
|
|
- * @param downloadFlag 下载标记,默认全部下载(0:全部下载,1:只下载图片,2:只下载视频)
|
|
|
- */
|
|
|
- @Override
|
|
|
- public void download(List<DouYinAnalysis> douYinAnalysisList, Integer flag, boolean repeatFlag, Integer downloadFlag) {
|
|
|
- try {
|
|
|
- // 开始时间
|
|
|
- long stime = System.currentTimeMillis();
|
|
|
- if (downloadFlag == null) {
|
|
|
- downloadFlag = 0;
|
|
|
- }
|
|
|
- log.info("download: 共获取到: {} 条数据", douYinAnalysisList.size());
|
|
|
- if (flag == 0) {
|
|
|
- DouYinAnalysis douYinAnalysis = douYinAnalysisList.get(0);
|
|
|
- log.info("download: 共获取到: {} 条数据, 当前为第: {} 条数据", douYinAnalysisList.size(), 1);
|
|
|
- if (videoCount(douYinAnalysis) < 1) {
|
|
|
- AnalysisLog analysisLog = addLog(douYinAnalysis);
|
|
|
- localCacheFile();
|
|
|
- List<String> downloadList = addAllDownloadLink(douYinAnalysis, downloadFlag);
|
|
|
- downloadFunction(downloadList, 0, analysisLog, "");
|
|
|
- } else {
|
|
|
- log.info("download: 解析单条视频: 当前视频已下载");
|
|
|
- }
|
|
|
- } else {
|
|
|
- //用户主页解析放到同一个文件夹
|
|
|
- String upLoadPath;
|
|
|
- int count = 0;
|
|
|
- for (DouYinAnalysis douYinAnalysis : douYinAnalysisList) {
|
|
|
- count++;
|
|
|
- log.info("download: 共获取到: {} 条数据, 当前为第: {} 条数据", douYinAnalysisList.size(), count);
|
|
|
- //全部下载或者增量下载
|
|
|
- if (repeatFlag && videoCount(douYinAnalysis) > 0) {
|
|
|
- //查重且多条, 跳过此条
|
|
|
- continue;
|
|
|
- }
|
|
|
- AnalysisLog analysisLog = addLog(douYinAnalysis);
|
|
|
- //第一次循环, 保证文件夹存在
|
|
|
- //用户昵称
|
|
|
- String nickname = douYinAnalysis.getNickname();
|
|
|
- //规范昵称
|
|
|
- nickname = StringUtil.getString(nickname);
|
|
|
- //保证都在文件夹下, 不然文件会有错误
|
|
|
- if (nickname == null || nickname.isEmpty()) {
|
|
|
- nickname = "未命名";
|
|
|
- }
|
|
|
- //放到user文件夹下
|
|
|
- upLoadPath = webDavUrl + "user/" + nickname + "/";
|
|
|
- if (count == 1) {
|
|
|
- WebDavJackRabbitUtil.init(webDavUrl, webDavUserName, webDavPassWord);
|
|
|
- //保证user根目录存在
|
|
|
- WebDavJackRabbitUtil.mkdir(webDavUrl + "user/");
|
|
|
- if (WebDavJackRabbitUtil.mkdir(upLoadPath)) {
|
|
|
- log.info("download: WebDav创建文件夹成功: {}", upLoadPath);
|
|
|
- }
|
|
|
- }
|
|
|
- localCacheFile();
|
|
|
- List<String> downloadList = addAllDownloadLink(douYinAnalysis, downloadFlag);
|
|
|
- downloadFunction(downloadList, 1, analysisLog, upLoadPath);
|
|
|
- }
|
|
|
- }
|
|
|
- // 结束时间
|
|
|
- long etime = System.currentTimeMillis();
|
|
|
- // 计算执行时间
|
|
|
- long time = etime - stime;
|
|
|
- log.info("downloadFunction: 成功, 执行时间" + time + "毫秒");
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("download: ERROR MSG={}&Stack={}", e.getMessage(), e.getStackTrace());
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**************************************Function**********************************************/
|
|
|
-
|
|
|
- /**
|
|
|
- * 查询视频是否存在
|
|
|
- *
|
|
|
- * @param douYinAnalysis 单条视频
|
|
|
- * @return 查询的视频数量
|
|
|
- */
|
|
|
- private Integer videoCount(DouYinAnalysis douYinAnalysis) {
|
|
|
- LambdaUpdateWrapper<AnalysisLog> queryWrapper = new LambdaUpdateWrapper<>();
|
|
|
- queryWrapper.eq(AnalysisLog::getLink, douYinAnalysis.getLink()).or().eq(AnalysisLog::getItemId, douYinAnalysis.getItemId()).eq(AnalysisLog::getDeleteFlag, 0);
|
|
|
- return analysisLogMapper.selectCount(queryWrapper);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 新增解析记录
|
|
|
- *
|
|
|
- * @param douYinAnalysis 视频数据
|
|
|
- * @return AnalysisLog
|
|
|
- */
|
|
|
- private AnalysisLog addLog(DouYinAnalysis douYinAnalysis) {
|
|
|
- AnalysisLog analysisLog = new AnalysisLog();
|
|
|
- analysisLog.setItemId(douYinAnalysis.getItemId());
|
|
|
- analysisLog.setNickname(douYinAnalysis.getNickname());
|
|
|
- analysisLog.setUid(douYinAnalysis.getUid());
|
|
|
- analysisLog.setLink(douYinAnalysis.getLink());
|
|
|
- analysisLog.setPosition(1);
|
|
|
- analysisLog.setSource(douYinAnalysis.getSource());
|
|
|
- //规范文件名
|
|
|
- String title = StringUtil.getString(douYinAnalysis.getDesc());
|
|
|
- analysisLog.setTitle(title);
|
|
|
- analysisLog.setType(douYinAnalysis.getType());
|
|
|
- return analysisLog;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 保证本地缓存文件夹存在
|
|
|
- * /Cache/douyin/
|
|
|
- */
|
|
|
- private void localCacheFile() {
|
|
|
- //保证本地缓存文件夹存在
|
|
|
- File file = new File(filePath);
|
|
|
- if (!file.exists()) {
|
|
|
- if (file.mkdirs()) {
|
|
|
- log.info("localCacheFile: 文件夹不存在, 创建了新文件夹");
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 添加所有要下载的链接
|
|
|
- *
|
|
|
- * @param douYinAnalysis 解析内容
|
|
|
- * @param downloadFlag 下载标记(0:全部下载,1:只下载图片,2:只下载视频)
|
|
|
- * @return list
|
|
|
- */
|
|
|
- private List<String> addAllDownloadLink(DouYinAnalysis douYinAnalysis, Integer downloadFlag) {
|
|
|
- List<String> downloadList = new ArrayList<>();
|
|
|
- if (downloadFlag == 0 || downloadFlag == 1) {
|
|
|
- //解析的图片链接
|
|
|
- if (null != douYinAnalysis.getImage_list() && !douYinAnalysis.getImage_list().isEmpty()) {
|
|
|
- downloadList.addAll(douYinAnalysis.getImage_list());
|
|
|
- }
|
|
|
- }
|
|
|
- if (downloadFlag == 0 || downloadFlag == 2) {
|
|
|
- //解析的视频链接
|
|
|
- if (null != douYinAnalysis.getVideo_list() && !douYinAnalysis.getVideo_list().isEmpty()) {
|
|
|
- downloadList.addAll(douYinAnalysis.getVideo_list());
|
|
|
- }
|
|
|
- }
|
|
|
- return downloadList;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 下载方法
|
|
|
- *
|
|
|
- * @param downloadList 下载的链接
|
|
|
- * @param uploadFlag 上传标记{0:单条解析上传,其他:主页解析上传}
|
|
|
- * @param analysisLog 解析记录
|
|
|
- * @param uploadPath 上传路径(用户单独文件夹)
|
|
|
- */
|
|
|
- private void downloadFunction(List<String> downloadList, Integer uploadFlag, AnalysisLog analysisLog, String uploadPath) {
|
|
|
- try {
|
|
|
- //判断上传路径
|
|
|
- if (uploadFlag == 0) {
|
|
|
- //单条解析,传到指定文件夹
|
|
|
- uploadPath = webDavUrl;
|
|
|
- }
|
|
|
- if (downloadList.size() > 1) {
|
|
|
- // 创建临时zip文件
|
|
|
- String lastFilePath = filePath + analysisLog.getTitle() + nowTime() + ".zip";
|
|
|
- if (lastFilePath.length() > 100) {
|
|
|
- lastFilePath = filePath + nowTime() + ".zip";
|
|
|
- }
|
|
|
- File zip = new File(lastFilePath);
|
|
|
- analysisLog.setFileName(zip.getName());
|
|
|
- ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(zip.toPath()));
|
|
|
- int i = 1;
|
|
|
- for (String s : downloadList) {
|
|
|
- log.info("downloadFunction: ItemId:{}, 总文件数:{}, 当前个数:{}", analysisLog.getItemId(), downloadList.size(), i);
|
|
|
- HttpURLConnection conn = conn(s);
|
|
|
- String fileType = new String(conn.getHeaderField("Content-Type").getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
|
|
|
- fileType = "." + fileType.substring(fileType.lastIndexOf("/") + 1);
|
|
|
- //得到输入流
|
|
|
- InputStream inputStream = conn.getInputStream();
|
|
|
- if (".webp".equalsIgnoreCase(fileType)) {
|
|
|
- inputStream = WebpConvertUtil.webpConvertWithStream(inputStream, "jpg");
|
|
|
- fileType = ".jpg";
|
|
|
- }
|
|
|
- if (".json".equalsIgnoreCase(fileType)) {
|
|
|
- throw new RuntimeException("勾八抖音又封IP啦~~~");
|
|
|
- }
|
|
|
- BufferedInputStream in = new BufferedInputStream(inputStream);
|
|
|
- out.putNextEntry(new ZipEntry(nowTime() + fileType));
|
|
|
- int b;
|
|
|
- while ((b = in.read()) != -1) {
|
|
|
- out.write(b);
|
|
|
- }
|
|
|
- in.close();
|
|
|
- i++;
|
|
|
- }
|
|
|
- out.close();
|
|
|
- //从本地上传到Nas
|
|
|
- if (WebDavUpload.upload(uploadPath, webDavUserName, webDavPassWord, lastFilePath)) {
|
|
|
- log.info("downloadFunction: WebDav上传成功");
|
|
|
- //删除本地文件
|
|
|
- log.info(zip.delete() ? "downloadFunction: 本地文件删除成功" : "downloadFunction: 本地文件删除失败");
|
|
|
- analysisLogMapper.insert(analysisLog);
|
|
|
- }
|
|
|
- } else if (downloadList.size() == 1) {
|
|
|
- //单个下载
|
|
|
- HttpURLConnection conn = conn(downloadList.get(0));
|
|
|
- String fileType = new String(conn.getHeaderField("Content-Type").getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
|
|
|
- fileType = "." + fileType.substring(fileType.lastIndexOf("/") + 1);
|
|
|
- //得到输入流
|
|
|
- InputStream inputStream = conn.getInputStream();
|
|
|
- if (".webp".equalsIgnoreCase(fileType)) {
|
|
|
- inputStream = WebpConvertUtil.webpConvertWithStream(inputStream, "jpg");
|
|
|
- fileType = ".jpg";
|
|
|
- }
|
|
|
- if (".json".equalsIgnoreCase(fileType)) {
|
|
|
- throw new RuntimeException("勾八抖音又封IP啦~~~");
|
|
|
- }
|
|
|
- String lastFilePath = filePath + analysisLog.getTitle() + nowTime() + fileType;
|
|
|
- if (lastFilePath.length() > 100) {
|
|
|
- lastFilePath = filePath + nowTime() + fileType;
|
|
|
- }
|
|
|
- File file = new File(lastFilePath);
|
|
|
- FileUtils.copyInputStreamToFile(inputStream, file);
|
|
|
- analysisLog.setFileName(file.getName());
|
|
|
- //从本地上传到Nas
|
|
|
- if (WebDavUpload.upload(uploadPath, webDavUserName, webDavPassWord, lastFilePath)) {
|
|
|
- log.info("downloadFunction: WebDav上传成功");
|
|
|
- //删除本地文件
|
|
|
- log.info(file.delete() ? "downloadFunction: 本地文件删除成功" : "downloadFunction: 本地文件删除失败");
|
|
|
- analysisLogMapper.insert(analysisLog);
|
|
|
- }
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("downloadFunction: ERROR MSG={}&Stack={}", e.getMessage(), e.getStackTrace());
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取Token和减少可用次数
|
|
|
- *
|
|
|
- * @return token
|
|
|
- */
|
|
|
- private String getToken() {
|
|
|
- String token = "";
|
|
|
- List<AnalysisUser> allUser = analysisUserService.getAllUser();
|
|
|
- for (AnalysisUser analysisUser : allUser) {
|
|
|
- if (Integer.parseInt(analysisUser.getRemainingTimes()) > 0) {
|
|
|
- AnalysisToken analysisToken = analysisTokenService.getOne(analysisUser.getId());
|
|
|
- token = analysisToken.getToken();
|
|
|
- //减少可用次数,调用失败可用次数也会减少
|
|
|
- int times = Integer.parseInt(analysisUser.getRemainingTimes()) - 1;
|
|
|
- analysisUser.setRemainingTimes(times + "");
|
|
|
- analysisUserService.updateAnalysisUser(analysisUser);
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- return token;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 现在时间,无符号到毫秒
|
|
|
- *
|
|
|
- * @return String
|
|
|
- */
|
|
|
- private String nowTime() {
|
|
|
- SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmssSSS");
|
|
|
- return formatter.format(new Date());
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * http链接
|
|
|
- *
|
|
|
- * @param link 解析地址
|
|
|
- * @return HttpURLConnection
|
|
|
- */
|
|
|
- private HttpURLConnection conn(String link) {
|
|
|
- HttpURLConnection conn = null;
|
|
|
- try {
|
|
|
- URL url = new URL(link);
|
|
|
- conn = (HttpURLConnection) url.openConnection();
|
|
|
- //设置超时间为3秒
|
|
|
- conn.setConnectTimeout(3 * 1000);
|
|
|
- //防止屏蔽程序抓取而返回403错误
|
|
|
- conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
|
|
|
- } catch (IOException e) {
|
|
|
- log.error(e.getMessage());
|
|
|
- }
|
|
|
- return conn;
|
|
|
- }
|
|
|
-}
|