Compare commits
10 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
3ddc94758d | |
|
|
0585546be0 | |
|
|
3aeb8b9a14 | |
|
|
ffe6931391 | |
|
|
3962b9ff2e | |
|
|
0a51d7e5e8 | |
|
|
2cbce1a574 | |
|
|
e60b827baa | |
|
|
64dd211839 | |
|
|
7f03ee969d |
|
|
@ -8,8 +8,7 @@ EXPOSE 24803
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# 创建 bin 目录
|
# 创建 bin 目录
|
||||||
RUN mkdir -p bin \
|
RUN mkdir -p config \
|
||||||
&& mkdir -p config \
|
|
||||||
&& mkdir -p lib \
|
&& mkdir -p lib \
|
||||||
&& mkdir -p log
|
&& mkdir -p log
|
||||||
|
|
||||||
|
|
|
||||||
24
README.md
24
README.md
|
|
@ -2,6 +2,13 @@
|
||||||
|
|
||||||
RSS阅读器,基于Java SpringBoot搭建。
|
RSS阅读器,基于Java SpringBoot搭建。
|
||||||
|
|
||||||
|
## 实现功能
|
||||||
|
|
||||||
|
1. RSS订阅功能;
|
||||||
|
2. 邮件推送;
|
||||||
|
3. 微博图片保存;
|
||||||
|
4. 微博图片上传AList;
|
||||||
|
|
||||||
## 部署运行
|
## 部署运行
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
|
|
@ -9,6 +16,7 @@ docker run -d --restart=always \
|
||||||
-v ./config:/app/config \
|
-v ./config:/app/config \
|
||||||
-v ./log:/app/log \
|
-v ./log:/app/log \
|
||||||
-v ./images:/app/images \
|
-v ./images:/app/images \
|
||||||
|
-v ./video:/app/video \
|
||||||
-v /etc/localtime:/etc/localtime:ro \
|
-v /etc/localtime:/etc/localtime:ro \
|
||||||
--name rss-reader \
|
--name rss-reader \
|
||||||
bcrjl/rss-reader:latest
|
bcrjl/rss-reader:latest
|
||||||
|
|
@ -21,8 +29,20 @@ bcrjl/rss-reader:latest
|
||||||
### 映射目录说明
|
### 映射目录说明
|
||||||
|
|
||||||
| 本地目录 | 容器目录 | 说明 |
|
| 本地目录 | 容器目录 | 说明 |
|
||||||
| --- | --- |-------|
|
|----------| --- |-------|
|
||||||
| ./config | /app/config | 配置文件 |
|
| ./config | /app/config | 配置文件 |
|
||||||
| ./log | /app/log | 运行日志 |
|
| ./log | /app/log | 运行日志 |
|
||||||
| ./images | /app/images | 下载的文件 |
|
| ./images | /app/images | 下载的图片 |
|
||||||
|
| ./video | /app/video | 下载的视频 |
|
||||||
|
|
||||||
|
## FAQ
|
||||||
|
|
||||||
|
### 1.AList上传异常
|
||||||
|
|
||||||
|
配置错误,请检查`config.setting`配置文件中的`aListUrl`配置,结尾不能带`/`
|
||||||
|
|
||||||
|
## TODO
|
||||||
|
|
||||||
|
1. 订阅请求增加代理配置
|
||||||
|
2推送渠道:企业微信、钉钉等
|
||||||
|
3...
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,8 @@
|
||||||
refresh=5
|
refresh=5
|
||||||
## 保存微博图片
|
## 保存微博图片
|
||||||
saveWeiBoImages=true
|
saveWeiBoImages=true
|
||||||
|
## 保存微博视频
|
||||||
|
saveWeiBoVideo=true
|
||||||
## 上传图片到AList
|
## 上传图片到AList
|
||||||
uploadAList=false
|
uploadAList=false
|
||||||
## AList Url
|
## AList Url
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ services:
|
||||||
- ./config:/app/config
|
- ./config:/app/config
|
||||||
- ./log:/app/log
|
- ./log:/app/log
|
||||||
- ./images:/app/images
|
- ./images:/app/images
|
||||||
|
- ./video:/app/video
|
||||||
- /etc/localtime:/etc/localtime:ro
|
- /etc/localtime:/etc/localtime:ro
|
||||||
environment:
|
environment:
|
||||||
- TZ=Asia/Shanghai
|
- TZ=Asia/Shanghai
|
||||||
|
|
|
||||||
11
pom.xml
11
pom.xml
|
|
@ -6,7 +6,7 @@
|
||||||
|
|
||||||
<groupId>com.bcrjl.rss</groupId>
|
<groupId>com.bcrjl.rss</groupId>
|
||||||
<artifactId>rss-reader</artifactId>
|
<artifactId>rss-reader</artifactId>
|
||||||
<version>0.4.0</version>
|
<version>0.5.1</version>
|
||||||
<name>RSS订阅阅读器</name>
|
<name>RSS订阅阅读器</name>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
|
|
@ -24,6 +24,7 @@
|
||||||
<lombok.version>1.18.34</lombok.version>
|
<lombok.version>1.18.34</lombok.version>
|
||||||
<hutool.version>5.8.29</hutool.version>
|
<hutool.version>5.8.29</hutool.version>
|
||||||
<javax.mail.version>1.6.2</javax.mail.version>
|
<javax.mail.version>1.6.2</javax.mail.version>
|
||||||
|
<commons.io.version>2.14.0</commons.io.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -50,6 +51,12 @@
|
||||||
<artifactId>spring-boot-starter-thymeleaf</artifactId>
|
<artifactId>spring-boot-starter-thymeleaf</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-test</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>cn.dev33</groupId>
|
<groupId>cn.dev33</groupId>
|
||||||
<artifactId>sa-token-spring-boot-starter</artifactId>
|
<artifactId>sa-token-spring-boot-starter</artifactId>
|
||||||
|
|
@ -71,7 +78,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>commons-io</groupId>
|
<groupId>commons-io</groupId>
|
||||||
<artifactId>commons-io</artifactId>
|
<artifactId>commons-io</artifactId>
|
||||||
<version>2.11.0</version>
|
<version>${commons.io.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<!-- HuTool -->
|
<!-- HuTool -->
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ public interface AppConstant {
|
||||||
*/
|
*/
|
||||||
String RSS_CONFIG_PATH = System.getProperty("user.dir") + "/config/data.json";
|
String RSS_CONFIG_PATH = System.getProperty("user.dir") + "/config/data.json";
|
||||||
String IMAGES_PATH = System.getProperty("user.dir") + "/images/";
|
String IMAGES_PATH = System.getProperty("user.dir") + "/images/";
|
||||||
|
String VIDEO_PATH = System.getProperty("user.dir") + "/video/";
|
||||||
|
|
||||||
String SET_SYSTEM = "system";
|
String SET_SYSTEM = "system";
|
||||||
String SET_MAIL = "mail";
|
String SET_MAIL = "mail";
|
||||||
|
|
@ -36,6 +37,11 @@ public interface AppConstant {
|
||||||
*/
|
*/
|
||||||
String SAVE_WEIBO_IMAGES = "saveWeiBoImages";
|
String SAVE_WEIBO_IMAGES = "saveWeiBoImages";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 保存微博视频
|
||||||
|
*/
|
||||||
|
String SAVE_WEIBO_VIDEO = "saveWeiBoVideo";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 上传图片到AList
|
* 上传图片到AList
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -20,18 +20,26 @@ import java.util.Map;
|
||||||
import static com.bcrjl.rss.common.constant.AppConstant.*;
|
import static com.bcrjl.rss.common.constant.AppConstant.*;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 功能描述:
|
* AList工具类
|
||||||
*
|
*
|
||||||
* @author yanqs
|
* @author yanqs
|
||||||
* @since 2024-08-10
|
|
||||||
*/
|
*/
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class AListUtils {
|
public class AListUtils {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AList 登录获取 Token 接口路径
|
||||||
|
*/
|
||||||
private static final String TOKEN_URL = "/api/auth/login";
|
private static final String TOKEN_URL = "/api/auth/login";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 上传流 接口路径
|
||||||
|
*/
|
||||||
private static final String UPLOAD_URL = "/api/fs/put";
|
private static final String UPLOAD_URL = "/api/fs/put";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建本地缓存
|
||||||
|
*/
|
||||||
private static TimedCache<String, String> timedCache = CacheUtil.newTimedCache(86400000);
|
private static TimedCache<String, String> timedCache = CacheUtil.newTimedCache(86400000);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -65,9 +73,15 @@ public class AListUtils {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 上传文件
|
||||||
|
*
|
||||||
|
* @param fileByte 文件字节
|
||||||
|
* @param fileName 文件名
|
||||||
|
*/
|
||||||
public static void uploadFile(byte[] fileByte, String fileName) {
|
public static void uploadFile(byte[] fileByte, String fileName) {
|
||||||
try {
|
try {
|
||||||
String time = DateUtil.format(new Date(), "yyyyMMddHH");
|
String time = DateUtil.format(new Date(), "yyyyMMdd");
|
||||||
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
||||||
Setting systemSetting = setting.getSetting(SET_SYSTEM);
|
Setting systemSetting = setting.getSetting(SET_SYSTEM);
|
||||||
String aListUrl = systemSetting.get(ALIST_URL);
|
String aListUrl = systemSetting.get(ALIST_URL);
|
||||||
|
|
@ -80,9 +94,7 @@ public class AListUtils {
|
||||||
.execute();
|
.execute();
|
||||||
if (httpResponse.isOk()) {
|
if (httpResponse.isOk()) {
|
||||||
String message = JSONUtil.parseObj(httpResponse.body()).getStr("message");
|
String message = JSONUtil.parseObj(httpResponse.body()).getStr("message");
|
||||||
if ("success".equals(message)) {
|
if (!"success".equals(message)) {
|
||||||
//log.info("AList上传成功");
|
|
||||||
} else {
|
|
||||||
log.info("AList上传失败:{}", message);
|
log.info("AList上传失败:{}", message);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
package com.bcrjl.rss.common.util;
|
|
||||||
|
|
||||||
import cn.hutool.http.Header;
|
|
||||||
import cn.hutool.http.HttpRequest;
|
|
||||||
import cn.hutool.http.HttpResponse;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.regex.Matcher;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
|
|
||||||
import static com.bcrjl.rss.common.constant.AppConstant.USER_AGENT;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Html 工具类
|
|
||||||
*
|
|
||||||
* @author yanqs
|
|
||||||
*/
|
|
||||||
@Slf4j
|
|
||||||
public class HtmlParseUtils {
|
|
||||||
/**
|
|
||||||
* 获取html中的图片
|
|
||||||
*
|
|
||||||
* @param htmlContent html内容
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
public static List<String> extractImageUrls(String htmlContent) {
|
|
||||||
List<String> imageUrls = new ArrayList<>();
|
|
||||||
String regex = "<img\\s+[^>]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>";
|
|
||||||
Pattern pattern = Pattern.compile(regex);
|
|
||||||
Matcher matcher = pattern.matcher(htmlContent);
|
|
||||||
while (matcher.find()) {
|
|
||||||
String imageUrl = matcher.group(1);
|
|
||||||
imageUrls.add(imageUrl);
|
|
||||||
}
|
|
||||||
return imageUrls;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取微博图片流文件
|
|
||||||
*
|
|
||||||
* @param fileName 微博图片名称
|
|
||||||
* @return HttpResponse
|
|
||||||
*/
|
|
||||||
public static HttpResponse getWeiBoImagesHttpRequest(String fileName) {
|
|
||||||
try {
|
|
||||||
String url = "https://tvax3.sinaimg.cn/large/" + fileName;
|
|
||||||
HttpRequest request = HttpRequest.get(url)
|
|
||||||
.header(Header.REFERER, "https://weibo.com/")
|
|
||||||
.header(Header.USER_AGENT, USER_AGENT)
|
|
||||||
.timeout(20000);
|
|
||||||
return request.executeAsync();
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("获取微博图片数据异常:", e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,113 @@
|
||||||
|
package com.bcrjl.rss.common.util;
|
||||||
|
|
||||||
|
import cn.hutool.http.Header;
|
||||||
|
import cn.hutool.http.HttpRequest;
|
||||||
|
import cn.hutool.http.HttpResponse;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import static com.bcrjl.rss.common.constant.AppConstant.USER_AGENT;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Html 工具类
|
||||||
|
*
|
||||||
|
* @author yanqs
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class HtmlUtils {
|
||||||
|
/**
|
||||||
|
* 获取html中的图片
|
||||||
|
*
|
||||||
|
* @param htmlContent html内容
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static List<String> extractImageUrls(String htmlContent) {
|
||||||
|
String regex = "<img\\s+[^>]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>";
|
||||||
|
return extractUrls(htmlContent, regex);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取html中的视频
|
||||||
|
*
|
||||||
|
* @param htmlContent html内容
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static List<String> extractVideoUrls(String htmlContent) {
|
||||||
|
String regex = "<source\\s+[^>]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>";
|
||||||
|
return extractUrls(htmlContent, regex);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取url中的文件名称
|
||||||
|
*
|
||||||
|
* @param url url
|
||||||
|
* @return 文件名称
|
||||||
|
*/
|
||||||
|
public static String getFileName(String url) {
|
||||||
|
int lastSlashIndex = url.lastIndexOf('/');
|
||||||
|
// 如果找到了斜杠,就从斜杠后面截取字符串
|
||||||
|
String fileName = url.substring(lastSlashIndex + 1);
|
||||||
|
int queryIndex = fileName.indexOf('?');
|
||||||
|
if (queryIndex == -1) {
|
||||||
|
return fileName;
|
||||||
|
} else {
|
||||||
|
return fileName.substring(0, queryIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取微博图片流文件
|
||||||
|
*
|
||||||
|
* @param fileName 微博图片名称
|
||||||
|
* @return HttpResponse
|
||||||
|
*/
|
||||||
|
public static HttpResponse getWeiBoImagesHttpRequest(String fileName) {
|
||||||
|
try {
|
||||||
|
String url = "https://tvax3.sinaimg.cn/large/" + fileName;
|
||||||
|
HttpRequest request = HttpRequest.get(url)
|
||||||
|
.header(Header.REFERER, "https://weibo.com/")
|
||||||
|
.header(Header.USER_AGENT, USER_AGENT)
|
||||||
|
.timeout(20000);
|
||||||
|
return request.executeAsync();
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("获取微博图片数据异常:", e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static HttpResponse getWeiBoVideoHttpRequest(String url) {
|
||||||
|
try {
|
||||||
|
HttpRequest request = HttpRequest.get(url)
|
||||||
|
.header(Header.REFERER, "https://weibo.com/")
|
||||||
|
.header(Header.USER_AGENT, USER_AGENT)
|
||||||
|
.timeout(20000);
|
||||||
|
return request.executeAsync();
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("获取视频数据异常:", e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 根据正则获取html中的内容
|
||||||
|
*
|
||||||
|
* @param htmlContent html内容
|
||||||
|
* @param regex 正则
|
||||||
|
* @return urls
|
||||||
|
*/
|
||||||
|
private static List<String> extractUrls(String htmlContent, String regex) {
|
||||||
|
List<String> urls = new ArrayList<>();
|
||||||
|
Pattern pattern = Pattern.compile(regex);
|
||||||
|
Matcher matcher = pattern.matcher(htmlContent);
|
||||||
|
while (matcher.find()) {
|
||||||
|
String url = matcher.group(1);
|
||||||
|
urls.add(url);
|
||||||
|
}
|
||||||
|
return urls;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -41,6 +41,8 @@ public class FileMonitor {
|
||||||
"refresh = 5\n" +
|
"refresh = 5\n" +
|
||||||
"## 保存微博图片\n" +
|
"## 保存微博图片\n" +
|
||||||
"saveWeiBoImages=false\n" +
|
"saveWeiBoImages=false\n" +
|
||||||
|
"## 保存微博视频\n" +
|
||||||
|
"saveWeiBoVideo=true\n" +
|
||||||
"## 上传图片到AList\n" +
|
"## 上传图片到AList\n" +
|
||||||
"uploadAList=false\n" +
|
"uploadAList=false\n" +
|
||||||
"## AList Url\n" +
|
"## AList Url\n" +
|
||||||
|
|
@ -50,7 +52,7 @@ public class FileMonitor {
|
||||||
"## AList 密码\n" +
|
"## AList 密码\n" +
|
||||||
"aListPass=\n" +
|
"aListPass=\n" +
|
||||||
"## AList 上传路径\n" +
|
"## AList 上传路径\n" +
|
||||||
"aListUploadPath=" +
|
"aListUploadPath=\n" +
|
||||||
"[mail]\n" +
|
"[mail]\n" +
|
||||||
"## 启用邮件推送\n" +
|
"## 启用邮件推送\n" +
|
||||||
"enable=false\n" +
|
"enable=false\n" +
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollUtil;
|
||||||
import cn.hutool.core.io.FileUtil;
|
import cn.hutool.core.io.FileUtil;
|
||||||
import cn.hutool.core.io.file.FileWriter;
|
import cn.hutool.core.io.file.FileWriter;
|
||||||
import cn.hutool.core.io.resource.ResourceUtil;
|
import cn.hutool.core.io.resource.ResourceUtil;
|
||||||
|
import cn.hutool.core.thread.ThreadUtil;
|
||||||
import cn.hutool.core.util.CharsetUtil;
|
import cn.hutool.core.util.CharsetUtil;
|
||||||
import cn.hutool.extra.mail.MailAccount;
|
import cn.hutool.extra.mail.MailAccount;
|
||||||
import cn.hutool.extra.mail.MailUtil;
|
import cn.hutool.extra.mail.MailUtil;
|
||||||
|
|
@ -12,7 +13,7 @@ import cn.hutool.json.JSONObject;
|
||||||
import cn.hutool.json.JSONUtil;
|
import cn.hutool.json.JSONUtil;
|
||||||
import cn.hutool.setting.Setting;
|
import cn.hutool.setting.Setting;
|
||||||
import com.bcrjl.rss.common.util.AListUtils;
|
import com.bcrjl.rss.common.util.AListUtils;
|
||||||
import com.bcrjl.rss.common.util.HtmlParseUtils;
|
import com.bcrjl.rss.common.util.HtmlUtils;
|
||||||
import com.bcrjl.rss.common.util.MailUtils;
|
import com.bcrjl.rss.common.util.MailUtils;
|
||||||
import com.bcrjl.rss.common.util.RssUtils;
|
import com.bcrjl.rss.common.util.RssUtils;
|
||||||
import com.bcrjl.rss.model.entity.RssEntity;
|
import com.bcrjl.rss.model.entity.RssEntity;
|
||||||
|
|
@ -90,7 +91,12 @@ public class RssJob {
|
||||||
if (CollUtil.isNotEmpty(list)) {
|
if (CollUtil.isNotEmpty(list)) {
|
||||||
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
||||||
Setting emailSetting = setting.getSetting(SET_MAIL);
|
Setting emailSetting = setting.getSetting(SET_MAIL);
|
||||||
saveWeiBoImagesOrUpdateAlist(list);
|
ThreadUtil.execAsync(new Runnable() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
saveWeiBoImagesOrUpdateAList(list);
|
||||||
|
}
|
||||||
|
});
|
||||||
if (emailSetting.getBool(MAIL_CONFIG_ENABLE) && emailSetting.getBool("sendUpdate")) {
|
if (emailSetting.getBool(MAIL_CONFIG_ENABLE) && emailSetting.getBool("sendUpdate")) {
|
||||||
// 如果邮箱开启且发送更新邮件开启 则推送通知
|
// 如果邮箱开启且发送更新邮件开启 则推送通知
|
||||||
StringBuffer stringBuffer = new StringBuffer();
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
|
@ -111,22 +117,20 @@ public class RssJob {
|
||||||
/**
|
/**
|
||||||
* 保存微博图片到本地且上传AList
|
* 保存微博图片到本地且上传AList
|
||||||
*/
|
*/
|
||||||
private void saveWeiBoImagesOrUpdateAlist(List<RssEntity> list) {
|
private void saveWeiBoImagesOrUpdateAList(List<RssEntity> list) {
|
||||||
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
||||||
Setting systemSetting = setting.getSetting(SET_SYSTEM);
|
Setting systemSetting = setting.getSetting(SET_SYSTEM);
|
||||||
Boolean saveImages = Boolean.valueOf(systemSetting.get(SAVE_WEIBO_IMAGES));
|
boolean saveImages = Boolean.parseBoolean(systemSetting.get(SAVE_WEIBO_IMAGES));
|
||||||
Boolean uploadAList = Boolean.valueOf(systemSetting.get(UPLOAD_ALIST));
|
boolean saveVideo = Boolean.valueOf(systemSetting.get(SAVE_WEIBO_VIDEO));
|
||||||
|
boolean uploadAList = Boolean.valueOf(systemSetting.get(UPLOAD_ALIST));
|
||||||
if (saveImages) {
|
if (saveImages) {
|
||||||
// 保存图片
|
// 保存图片
|
||||||
list.forEach(obj -> {
|
list.forEach(obj -> {
|
||||||
List<String> imgList = HtmlParseUtils.extractImageUrls(obj.getDescription());
|
List<String> imgList = HtmlUtils.extractImageUrls(obj.getDescription());
|
||||||
imgList.forEach(imgObj -> {
|
imgList.forEach(imgObj -> {
|
||||||
if (imgObj.contains("sinaimg") && !imgObj.contains("timeline_card") && !imgObj.contains("qixi2018")) {
|
if (imgObj.contains("sinaimg") && !imgObj.contains("timeline_card") && !imgObj.contains("qixi2018")) {
|
||||||
int lastSlashIndex = imgObj.lastIndexOf('/');
|
String fileName = HtmlUtils.getFileName(imgObj);
|
||||||
// 如果找到了斜杠,就从斜杠后面截取字符串
|
HttpResponse weiBoImagesHttpRequest = HtmlUtils.getWeiBoImagesHttpRequest(fileName);
|
||||||
String fileName = imgObj.substring(lastSlashIndex + 1);
|
|
||||||
//log.info("微博图片文件名:{}", fileName);
|
|
||||||
HttpResponse weiBoImagesHttpRequest = HtmlParseUtils.getWeiBoImagesHttpRequest(fileName);
|
|
||||||
byte[] bytes = weiBoImagesHttpRequest.bodyBytes();
|
byte[] bytes = weiBoImagesHttpRequest.bodyBytes();
|
||||||
FileUtil.writeBytes(bytes, new File(IMAGES_PATH + fileName));
|
FileUtil.writeBytes(bytes, new File(IMAGES_PATH + fileName));
|
||||||
if (uploadAList) {
|
if (uploadAList) {
|
||||||
|
|
@ -136,6 +140,23 @@ public class RssJob {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
if(saveVideo){
|
||||||
|
// 保存视频
|
||||||
|
list.forEach(obj -> {
|
||||||
|
List<String> videoList = HtmlUtils.extractVideoUrls(obj.getDescription());
|
||||||
|
if(CollUtil.isNotEmpty(videoList)){
|
||||||
|
videoList.forEach(videoObj -> {
|
||||||
|
String fileName = HtmlUtils.getFileName(videoObj);
|
||||||
|
HttpResponse weiBoVideoHttpRequest = HtmlUtils.getWeiBoVideoHttpRequest(videoObj);
|
||||||
|
byte[] bytes = weiBoVideoHttpRequest.bodyBytes();
|
||||||
|
FileUtil.writeBytes(bytes, new File(VIDEO_PATH + fileName));
|
||||||
|
if (uploadAList) {
|
||||||
|
AListUtils.uploadFile(bytes, fileName);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,51 +0,0 @@
|
||||||
import java.io.BufferedReader;
|
|
||||||
import java.io.FileReader;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.regex.Matcher;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Html测试
|
|
||||||
*
|
|
||||||
* @author yanqs
|
|
||||||
* @since 2024-08-10
|
|
||||||
*/
|
|
||||||
public class HtmlTest {
|
|
||||||
public static void main(String[] args) throws IOException{
|
|
||||||
String str="为什么温泉♨️水那么黄? <img style=\"\" src=\"https://tvax1.sinaimg.cn/large/00759jQJly1hsg8uwgmavj31401hcdy4.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uvtqz7j31401hck91.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax2.sinaimg.cn/large/00759jQJly1hsg8ux1psqj31401z44h1.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8uxycmvj31401hc16o.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax2.sinaimg.cn/large/00759jQJly1hsg8uyn21kj31401hc4gl.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8uzeqdzj31401hck84.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uzyy3yj3140140gz4.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uv9nptj31401hctma.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8v09z04j31401404by.jpg\" referrerpolicy=\"no-referrer\"><br><br>";
|
|
||||||
List<String> strings = extractImageUrls(str);
|
|
||||||
strings.forEach(obj->{
|
|
||||||
System.out.println(obj);
|
|
||||||
});
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public static String readHtmlFile(String filePath) throws IOException {
|
|
||||||
StringBuilder content = new StringBuilder();
|
|
||||||
BufferedReader reader = new BufferedReader(new FileReader(filePath));
|
|
||||||
String line;
|
|
||||||
while ((line = reader.readLine()) != null) {
|
|
||||||
content.append(line);
|
|
||||||
}
|
|
||||||
reader.close();
|
|
||||||
return content.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<String> extractImageUrls(String htmlContent) {
|
|
||||||
List<String> imageUrls = new ArrayList<>();
|
|
||||||
String regex = "<img\\s+[^>]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>";
|
|
||||||
Pattern pattern = Pattern.compile(regex);
|
|
||||||
Matcher matcher = pattern.matcher(htmlContent);
|
|
||||||
while (matcher.find()) {
|
|
||||||
String imageUrl = matcher.group(1);
|
|
||||||
imageUrls.add(imageUrl);
|
|
||||||
}
|
|
||||||
return imageUrls;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,40 +0,0 @@
|
||||||
import org.w3c.dom.Document;
|
|
||||||
import org.w3c.dom.Element;
|
|
||||||
import org.w3c.dom.NodeList;
|
|
||||||
|
|
||||||
import javax.xml.parsers.DocumentBuilder;
|
|
||||||
import javax.xml.parsers.DocumentBuilderFactory;
|
|
||||||
import java.net.URL;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* RSS获取 测试类
|
|
||||||
*
|
|
||||||
* @author yanqs
|
|
||||||
*/
|
|
||||||
public class RSSReader {
|
|
||||||
public static void main(String[] args) throws Exception {
|
|
||||||
// RSS feed URL
|
|
||||||
// URL rssUrl = new URL("https://rsshub.ys.bcrjl.com/weibo/user/6489032761");
|
|
||||||
URL rssUrl = new URL("https://blog.yanqingshan.com/feed/");
|
|
||||||
|
|
||||||
// Create a DocumentBuilder
|
|
||||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
|
||||||
DocumentBuilder builder = factory.newDocumentBuilder();
|
|
||||||
|
|
||||||
// Parse the RSS file
|
|
||||||
Document document = builder.parse(rssUrl.openStream());
|
|
||||||
|
|
||||||
// Get all items
|
|
||||||
NodeList items = document.getElementsByTagName("item");
|
|
||||||
|
|
||||||
for (int i = 0; i < items.getLength(); i++) {
|
|
||||||
Element item = (Element) items.item(i);
|
|
||||||
Element title = (Element) item.getElementsByTagName("title").item(0);
|
|
||||||
Element link = (Element) item.getElementsByTagName("link").item(0);
|
|
||||||
|
|
||||||
// Print the title
|
|
||||||
System.out.println(title.getTextContent());
|
|
||||||
System.out.println(link.getTextContent());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,69 @@
|
||||||
|
package com.bcrjl.rss.common.util;
|
||||||
|
|
||||||
|
import cn.hutool.core.io.FileUtil;
|
||||||
|
import cn.hutool.http.Header;
|
||||||
|
import cn.hutool.http.HttpRequest;
|
||||||
|
import cn.hutool.http.HttpResponse;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static com.bcrjl.rss.common.constant.AppConstant.IMAGES_PATH;
|
||||||
|
import static com.bcrjl.rss.common.constant.AppConstant.USER_AGENT;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Html 工具类测试
|
||||||
|
*
|
||||||
|
* @author yanqs
|
||||||
|
*/
|
||||||
|
class HtmlUtilsTest {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取html中的图片
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void extractImageUrls() {
|
||||||
|
String str = "为什么温泉♨️水那么黄? <img style=\"\" src=\"https://tvax1.sinaimg.cn/large/00759jQJly1hsg8uwgmavj31401hcdy4.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uvtqz7j31401hck91.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax2.sinaimg.cn/large/00759jQJly1hsg8ux1psqj31401z44h1.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8uxycmvj31401hc16o.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax2.sinaimg.cn/large/00759jQJly1hsg8uyn21kj31401hc4gl.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8uzeqdzj31401hck84.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uzyy3yj3140140gz4.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uv9nptj31401hctma.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8v09z04j31401404by.jpg\" referrerpolicy=\"no-referrer\"><br><br>";
|
||||||
|
List<String> strings = HtmlUtils.extractImageUrls(str);
|
||||||
|
strings.forEach(System.out::println);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 测试获取文件名
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void getFileName() {
|
||||||
|
String imgSrc = "https://h5.sinaimg.cn/m/emoticon/icon/others/ct_kele-4ce616ef95.png";
|
||||||
|
String videoSrc = "https://f.video.weibocdn.com/o0/H5AopYlvlx08hlyjma2A010412003vHq0E010.mp4?label=mp4_ld&template=360x480.24.0&ori=0&ps=1CwnkDw1GXwCQx&Expires=1724122135&ssig=QYOiNA1GMH&KID=unistore,video";
|
||||||
|
System.out.println(HtmlUtils.getFileName(imgSrc));
|
||||||
|
System.out.println(HtmlUtils.getFileName(videoSrc));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取微博图片流
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void getWeiBoImagesHttpRequest() {
|
||||||
|
String url="https://f.video.weibocdn.com/o0/Eglhl42Glx08hlnyHMZa01041200fr9J0E010.mp4?label=mp4_720p&template=720x1280.24.0&ori=0&ps=1CwnkDw1GXwCQx&Expires=1724122135&ssig=KmPchkXSzw&KID=unistore,video";
|
||||||
|
HttpResponse weiBoVideoHttpRequest = HtmlUtils.getWeiBoVideoHttpRequest(url);
|
||||||
|
byte[] bytes = weiBoVideoHttpRequest.bodyBytes();
|
||||||
|
FileUtil.writeBytes(bytes, new File(IMAGES_PATH + "Eglhl42Glx08hlnyHMZa01041200fr9J0E010.mp4"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取微博视频流
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void getWeiBoVideosHttpRequest() {
|
||||||
|
String url="https://f.video.weibocdn.com/o0/Eglhl42Glx08hlnyHMZa01041200fr9J0E010.mp4?label=mp4_720p&template=720x1280.24.0&ori=0&ps=1CwnkDw1GXwCQx&Expires=1724122135&ssig=KmPchkXSzw&KID=unistore,video";
|
||||||
|
HttpRequest request = HttpRequest.get(url)
|
||||||
|
.header(Header.REFERER, "https://weibo.com/")
|
||||||
|
.header(Header.USER_AGENT, USER_AGENT)
|
||||||
|
.timeout(20000);
|
||||||
|
HttpResponse httpResponse = request.executeAsync();
|
||||||
|
byte[] bytes = httpResponse.bodyBytes();
|
||||||
|
FileUtil.writeBytes(bytes, new File(IMAGES_PATH + "Eglhl42Glx08hlnyHMZa01041200fr9J0E010.mp4"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,13 +1,21 @@
|
||||||
|
package com.bcrjl.rss.common.util;
|
||||||
|
|
||||||
import cn.hutool.core.lang.Console;
|
import cn.hutool.core.lang.Console;
|
||||||
import cn.hutool.extra.mail.MailAccount;
|
import cn.hutool.extra.mail.MailAccount;
|
||||||
import com.bcrjl.rss.common.util.MailUtils;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* 邮件测试
|
||||||
|
*
|
||||||
* @author yanqs
|
* @author yanqs
|
||||||
* @since 2024-08-09
|
|
||||||
*/
|
*/
|
||||||
public class MailTest {
|
class MailUtilsTest {
|
||||||
public static void main(String[] args) {
|
|
||||||
|
/**
|
||||||
|
* 初始化邮件账号
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void initMailAccount() {
|
||||||
MailAccount mailAccount = MailUtils.initMailAccount();
|
MailAccount mailAccount = MailUtils.initMailAccount();
|
||||||
Console.log(mailAccount);
|
Console.log(mailAccount);
|
||||||
}
|
}
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
package com.bcrjl.rss.common.util;
|
||||||
|
|
||||||
|
import cn.hutool.core.lang.Console;
|
||||||
|
import com.bcrjl.rss.model.entity.RssEntity;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* RSSUtilsTest
|
||||||
|
*
|
||||||
|
* @author yanqs
|
||||||
|
*/
|
||||||
|
class RssUtilsTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void getRssList() {
|
||||||
|
String url = "https://blog.yanqingshan.com/feed/";
|
||||||
|
List<RssEntity> rssList = RssUtils.getRssList(url);
|
||||||
|
rssList.forEach(obj -> {
|
||||||
|
Console.log(obj);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue