Compare commits
10 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
3ddc94758d | |
|
|
0585546be0 | |
|
|
3aeb8b9a14 | |
|
|
ffe6931391 | |
|
|
3962b9ff2e | |
|
|
0a51d7e5e8 | |
|
|
2cbce1a574 | |
|
|
e60b827baa | |
|
|
64dd211839 | |
|
|
7f03ee969d |
|
|
@ -8,8 +8,7 @@ EXPOSE 24803
|
|||
WORKDIR /app
|
||||
|
||||
# 创建 bin 目录
|
||||
RUN mkdir -p bin \
|
||||
&& mkdir -p config \
|
||||
RUN mkdir -p config \
|
||||
&& mkdir -p lib \
|
||||
&& mkdir -p log
|
||||
|
||||
|
|
|
|||
28
README.md
28
README.md
|
|
@ -2,6 +2,13 @@
|
|||
|
||||
RSS阅读器,基于Java SpringBoot搭建。
|
||||
|
||||
## 实现功能
|
||||
|
||||
1. RSS订阅功能;
|
||||
2. 邮件推送;
|
||||
3. 微博图片保存;
|
||||
4. 微博图片上传AList;
|
||||
|
||||
## 部署运行
|
||||
|
||||
```shell
|
||||
|
|
@ -9,6 +16,7 @@ docker run -d --restart=always \
|
|||
-v ./config:/app/config \
|
||||
-v ./log:/app/log \
|
||||
-v ./images:/app/images \
|
||||
-v ./video:/app/video \
|
||||
-v /etc/localtime:/etc/localtime:ro \
|
||||
--name rss-reader \
|
||||
bcrjl/rss-reader:latest
|
||||
|
|
@ -20,9 +28,21 @@ bcrjl/rss-reader:latest
|
|||
|
||||
### 映射目录说明
|
||||
|
||||
| 本地目录 | 容器目录 | 说明 |
|
||||
| --- | --- |-------|
|
||||
| 本地目录 | 容器目录 | 说明 |
|
||||
|----------| --- |-------|
|
||||
| ./config | /app/config | 配置文件 |
|
||||
| ./log | /app/log | 运行日志 |
|
||||
| ./images | /app/images | 下载的文件 |
|
||||
| ./log | /app/log | 运行日志 |
|
||||
| ./images | /app/images | 下载的图片 |
|
||||
| ./video | /app/video | 下载的视频 |
|
||||
|
||||
## FAQ
|
||||
|
||||
### 1.AList上传异常
|
||||
|
||||
配置错误,请检查`config.setting`配置文件中的`aListUrl`配置,结尾不能带`/`
|
||||
|
||||
## TODO
|
||||
|
||||
1. 订阅请求增加代理配置
|
||||
2推送渠道:企业微信、钉钉等
|
||||
3...
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
refresh=5
|
||||
## 保存微博图片
|
||||
saveWeiBoImages=true
|
||||
## 保存微博视频
|
||||
saveWeiBoVideo=true
|
||||
## 上传图片到AList
|
||||
uploadAList=false
|
||||
## AList Url
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ services:
|
|||
- ./config:/app/config
|
||||
- ./log:/app/log
|
||||
- ./images:/app/images
|
||||
- ./video:/app/video
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
environment:
|
||||
- TZ=Asia/Shanghai
|
||||
|
|
|
|||
11
pom.xml
11
pom.xml
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
<groupId>com.bcrjl.rss</groupId>
|
||||
<artifactId>rss-reader</artifactId>
|
||||
<version>0.4.0</version>
|
||||
<version>0.5.1</version>
|
||||
<name>RSS订阅阅读器</name>
|
||||
|
||||
<properties>
|
||||
|
|
@ -24,6 +24,7 @@
|
|||
<lombok.version>1.18.34</lombok.version>
|
||||
<hutool.version>5.8.29</hutool.version>
|
||||
<javax.mail.version>1.6.2</javax.mail.version>
|
||||
<commons.io.version>2.14.0</commons.io.version>
|
||||
</properties>
|
||||
|
||||
|
||||
|
|
@ -50,6 +51,12 @@
|
|||
<artifactId>spring-boot-starter-thymeleaf</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>cn.dev33</groupId>
|
||||
<artifactId>sa-token-spring-boot-starter</artifactId>
|
||||
|
|
@ -71,7 +78,7 @@
|
|||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>2.11.0</version>
|
||||
<version>${commons.io.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- HuTool -->
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ public interface AppConstant {
|
|||
*/
|
||||
String RSS_CONFIG_PATH = System.getProperty("user.dir") + "/config/data.json";
|
||||
String IMAGES_PATH = System.getProperty("user.dir") + "/images/";
|
||||
String VIDEO_PATH = System.getProperty("user.dir") + "/video/";
|
||||
|
||||
String SET_SYSTEM = "system";
|
||||
String SET_MAIL = "mail";
|
||||
|
|
@ -36,6 +37,11 @@ public interface AppConstant {
|
|||
*/
|
||||
String SAVE_WEIBO_IMAGES = "saveWeiBoImages";
|
||||
|
||||
/**
|
||||
* 保存微博视频
|
||||
*/
|
||||
String SAVE_WEIBO_VIDEO = "saveWeiBoVideo";
|
||||
|
||||
/**
|
||||
* 上传图片到AList
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -20,18 +20,26 @@ import java.util.Map;
|
|||
import static com.bcrjl.rss.common.constant.AppConstant.*;
|
||||
|
||||
/**
|
||||
* 功能描述:
|
||||
* AList工具类
|
||||
*
|
||||
* @author yanqs
|
||||
* @since 2024-08-10
|
||||
*/
|
||||
@Slf4j
|
||||
public class AListUtils {
|
||||
|
||||
/**
|
||||
* AList 登录获取 Token 接口路径
|
||||
*/
|
||||
private static final String TOKEN_URL = "/api/auth/login";
|
||||
|
||||
/**
|
||||
* 上传流 接口路径
|
||||
*/
|
||||
private static final String UPLOAD_URL = "/api/fs/put";
|
||||
|
||||
/**
|
||||
* 创建本地缓存
|
||||
*/
|
||||
private static TimedCache<String, String> timedCache = CacheUtil.newTimedCache(86400000);
|
||||
|
||||
/**
|
||||
|
|
@ -65,9 +73,15 @@ public class AListUtils {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 上传文件
|
||||
*
|
||||
* @param fileByte 文件字节
|
||||
* @param fileName 文件名
|
||||
*/
|
||||
public static void uploadFile(byte[] fileByte, String fileName) {
|
||||
try {
|
||||
String time = DateUtil.format(new Date(), "yyyyMMddHH");
|
||||
String time = DateUtil.format(new Date(), "yyyyMMdd");
|
||||
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
||||
Setting systemSetting = setting.getSetting(SET_SYSTEM);
|
||||
String aListUrl = systemSetting.get(ALIST_URL);
|
||||
|
|
@ -80,9 +94,7 @@ public class AListUtils {
|
|||
.execute();
|
||||
if (httpResponse.isOk()) {
|
||||
String message = JSONUtil.parseObj(httpResponse.body()).getStr("message");
|
||||
if ("success".equals(message)) {
|
||||
//log.info("AList上传成功");
|
||||
} else {
|
||||
if (!"success".equals(message)) {
|
||||
log.info("AList上传失败:{}", message);
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1,59 +0,0 @@
|
|||
package com.bcrjl.rss.common.util;
|
||||
|
||||
import cn.hutool.http.Header;
|
||||
import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static com.bcrjl.rss.common.constant.AppConstant.USER_AGENT;
|
||||
|
||||
/**
|
||||
* Html 工具类
|
||||
*
|
||||
* @author yanqs
|
||||
*/
|
||||
@Slf4j
|
||||
public class HtmlParseUtils {
|
||||
/**
|
||||
* 获取html中的图片
|
||||
*
|
||||
* @param htmlContent html内容
|
||||
* @return
|
||||
*/
|
||||
public static List<String> extractImageUrls(String htmlContent) {
|
||||
List<String> imageUrls = new ArrayList<>();
|
||||
String regex = "<img\\s+[^>]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>";
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
Matcher matcher = pattern.matcher(htmlContent);
|
||||
while (matcher.find()) {
|
||||
String imageUrl = matcher.group(1);
|
||||
imageUrls.add(imageUrl);
|
||||
}
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取微博图片流文件
|
||||
*
|
||||
* @param fileName 微博图片名称
|
||||
* @return HttpResponse
|
||||
*/
|
||||
public static HttpResponse getWeiBoImagesHttpRequest(String fileName) {
|
||||
try {
|
||||
String url = "https://tvax3.sinaimg.cn/large/" + fileName;
|
||||
HttpRequest request = HttpRequest.get(url)
|
||||
.header(Header.REFERER, "https://weibo.com/")
|
||||
.header(Header.USER_AGENT, USER_AGENT)
|
||||
.timeout(20000);
|
||||
return request.executeAsync();
|
||||
} catch (Exception e) {
|
||||
log.error("获取微博图片数据异常:", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
package com.bcrjl.rss.common.util;
|
||||
|
||||
import cn.hutool.http.Header;
|
||||
import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static com.bcrjl.rss.common.constant.AppConstant.USER_AGENT;
|
||||
|
||||
/**
|
||||
* Html 工具类
|
||||
*
|
||||
* @author yanqs
|
||||
*/
|
||||
@Slf4j
|
||||
public class HtmlUtils {
|
||||
/**
|
||||
* 获取html中的图片
|
||||
*
|
||||
* @param htmlContent html内容
|
||||
* @return
|
||||
*/
|
||||
public static List<String> extractImageUrls(String htmlContent) {
|
||||
String regex = "<img\\s+[^>]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>";
|
||||
return extractUrls(htmlContent, regex);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取html中的视频
|
||||
*
|
||||
* @param htmlContent html内容
|
||||
* @return
|
||||
*/
|
||||
public static List<String> extractVideoUrls(String htmlContent) {
|
||||
String regex = "<source\\s+[^>]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>";
|
||||
return extractUrls(htmlContent, regex);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取url中的文件名称
|
||||
*
|
||||
* @param url url
|
||||
* @return 文件名称
|
||||
*/
|
||||
public static String getFileName(String url) {
|
||||
int lastSlashIndex = url.lastIndexOf('/');
|
||||
// 如果找到了斜杠,就从斜杠后面截取字符串
|
||||
String fileName = url.substring(lastSlashIndex + 1);
|
||||
int queryIndex = fileName.indexOf('?');
|
||||
if (queryIndex == -1) {
|
||||
return fileName;
|
||||
} else {
|
||||
return fileName.substring(0, queryIndex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取微博图片流文件
|
||||
*
|
||||
* @param fileName 微博图片名称
|
||||
* @return HttpResponse
|
||||
*/
|
||||
public static HttpResponse getWeiBoImagesHttpRequest(String fileName) {
|
||||
try {
|
||||
String url = "https://tvax3.sinaimg.cn/large/" + fileName;
|
||||
HttpRequest request = HttpRequest.get(url)
|
||||
.header(Header.REFERER, "https://weibo.com/")
|
||||
.header(Header.USER_AGENT, USER_AGENT)
|
||||
.timeout(20000);
|
||||
return request.executeAsync();
|
||||
} catch (Exception e) {
|
||||
log.error("获取微博图片数据异常:", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static HttpResponse getWeiBoVideoHttpRequest(String url) {
|
||||
try {
|
||||
HttpRequest request = HttpRequest.get(url)
|
||||
.header(Header.REFERER, "https://weibo.com/")
|
||||
.header(Header.USER_AGENT, USER_AGENT)
|
||||
.timeout(20000);
|
||||
return request.executeAsync();
|
||||
} catch (Exception e) {
|
||||
log.error("获取视频数据异常:", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 根据正则获取html中的内容
|
||||
*
|
||||
* @param htmlContent html内容
|
||||
* @param regex 正则
|
||||
* @return urls
|
||||
*/
|
||||
private static List<String> extractUrls(String htmlContent, String regex) {
|
||||
List<String> urls = new ArrayList<>();
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
Matcher matcher = pattern.matcher(htmlContent);
|
||||
while (matcher.find()) {
|
||||
String url = matcher.group(1);
|
||||
urls.add(url);
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
}
|
||||
|
|
@ -41,6 +41,8 @@ public class FileMonitor {
|
|||
"refresh = 5\n" +
|
||||
"## 保存微博图片\n" +
|
||||
"saveWeiBoImages=false\n" +
|
||||
"## 保存微博视频\n" +
|
||||
"saveWeiBoVideo=true\n" +
|
||||
"## 上传图片到AList\n" +
|
||||
"uploadAList=false\n" +
|
||||
"## AList Url\n" +
|
||||
|
|
@ -50,7 +52,7 @@ public class FileMonitor {
|
|||
"## AList 密码\n" +
|
||||
"aListPass=\n" +
|
||||
"## AList 上传路径\n" +
|
||||
"aListUploadPath=" +
|
||||
"aListUploadPath=\n" +
|
||||
"[mail]\n" +
|
||||
"## 启用邮件推送\n" +
|
||||
"enable=false\n" +
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollUtil;
|
|||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.io.file.FileWriter;
|
||||
import cn.hutool.core.io.resource.ResourceUtil;
|
||||
import cn.hutool.core.thread.ThreadUtil;
|
||||
import cn.hutool.core.util.CharsetUtil;
|
||||
import cn.hutool.extra.mail.MailAccount;
|
||||
import cn.hutool.extra.mail.MailUtil;
|
||||
|
|
@ -12,7 +13,7 @@ import cn.hutool.json.JSONObject;
|
|||
import cn.hutool.json.JSONUtil;
|
||||
import cn.hutool.setting.Setting;
|
||||
import com.bcrjl.rss.common.util.AListUtils;
|
||||
import com.bcrjl.rss.common.util.HtmlParseUtils;
|
||||
import com.bcrjl.rss.common.util.HtmlUtils;
|
||||
import com.bcrjl.rss.common.util.MailUtils;
|
||||
import com.bcrjl.rss.common.util.RssUtils;
|
||||
import com.bcrjl.rss.model.entity.RssEntity;
|
||||
|
|
@ -90,7 +91,12 @@ public class RssJob {
|
|||
if (CollUtil.isNotEmpty(list)) {
|
||||
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
||||
Setting emailSetting = setting.getSetting(SET_MAIL);
|
||||
saveWeiBoImagesOrUpdateAlist(list);
|
||||
ThreadUtil.execAsync(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
saveWeiBoImagesOrUpdateAList(list);
|
||||
}
|
||||
});
|
||||
if (emailSetting.getBool(MAIL_CONFIG_ENABLE) && emailSetting.getBool("sendUpdate")) {
|
||||
// 如果邮箱开启且发送更新邮件开启 则推送通知
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
|
|
@ -111,22 +117,20 @@ public class RssJob {
|
|||
/**
|
||||
* 保存微博图片到本地且上传AList
|
||||
*/
|
||||
private void saveWeiBoImagesOrUpdateAlist(List<RssEntity> list) {
|
||||
private void saveWeiBoImagesOrUpdateAList(List<RssEntity> list) {
|
||||
Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true);
|
||||
Setting systemSetting = setting.getSetting(SET_SYSTEM);
|
||||
Boolean saveImages = Boolean.valueOf(systemSetting.get(SAVE_WEIBO_IMAGES));
|
||||
Boolean uploadAList = Boolean.valueOf(systemSetting.get(UPLOAD_ALIST));
|
||||
boolean saveImages = Boolean.parseBoolean(systemSetting.get(SAVE_WEIBO_IMAGES));
|
||||
boolean saveVideo = Boolean.valueOf(systemSetting.get(SAVE_WEIBO_VIDEO));
|
||||
boolean uploadAList = Boolean.valueOf(systemSetting.get(UPLOAD_ALIST));
|
||||
if (saveImages) {
|
||||
// 保存图片
|
||||
list.forEach(obj -> {
|
||||
List<String> imgList = HtmlParseUtils.extractImageUrls(obj.getDescription());
|
||||
List<String> imgList = HtmlUtils.extractImageUrls(obj.getDescription());
|
||||
imgList.forEach(imgObj -> {
|
||||
if (imgObj.contains("sinaimg") && !imgObj.contains("timeline_card") && !imgObj.contains("qixi2018")) {
|
||||
int lastSlashIndex = imgObj.lastIndexOf('/');
|
||||
// 如果找到了斜杠,就从斜杠后面截取字符串
|
||||
String fileName = imgObj.substring(lastSlashIndex + 1);
|
||||
//log.info("微博图片文件名:{}", fileName);
|
||||
HttpResponse weiBoImagesHttpRequest = HtmlParseUtils.getWeiBoImagesHttpRequest(fileName);
|
||||
String fileName = HtmlUtils.getFileName(imgObj);
|
||||
HttpResponse weiBoImagesHttpRequest = HtmlUtils.getWeiBoImagesHttpRequest(fileName);
|
||||
byte[] bytes = weiBoImagesHttpRequest.bodyBytes();
|
||||
FileUtil.writeBytes(bytes, new File(IMAGES_PATH + fileName));
|
||||
if (uploadAList) {
|
||||
|
|
@ -136,6 +140,23 @@ public class RssJob {
|
|||
});
|
||||
});
|
||||
}
|
||||
if(saveVideo){
|
||||
// 保存视频
|
||||
list.forEach(obj -> {
|
||||
List<String> videoList = HtmlUtils.extractVideoUrls(obj.getDescription());
|
||||
if(CollUtil.isNotEmpty(videoList)){
|
||||
videoList.forEach(videoObj -> {
|
||||
String fileName = HtmlUtils.getFileName(videoObj);
|
||||
HttpResponse weiBoVideoHttpRequest = HtmlUtils.getWeiBoVideoHttpRequest(videoObj);
|
||||
byte[] bytes = weiBoVideoHttpRequest.bodyBytes();
|
||||
FileUtil.writeBytes(bytes, new File(VIDEO_PATH + fileName));
|
||||
if (uploadAList) {
|
||||
AListUtils.uploadFile(bytes, fileName);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,51 +0,0 @@
|
|||
import java.io.BufferedReader;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Html测试
|
||||
*
|
||||
* @author yanqs
|
||||
* @since 2024-08-10
|
||||
*/
|
||||
public class HtmlTest {
|
||||
public static void main(String[] args) throws IOException{
|
||||
String str="为什么温泉♨️水那么黄? <img style=\"\" src=\"https://tvax1.sinaimg.cn/large/00759jQJly1hsg8uwgmavj31401hcdy4.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uvtqz7j31401hck91.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax2.sinaimg.cn/large/00759jQJly1hsg8ux1psqj31401z44h1.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8uxycmvj31401hc16o.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax2.sinaimg.cn/large/00759jQJly1hsg8uyn21kj31401hc4gl.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8uzeqdzj31401hck84.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uzyy3yj3140140gz4.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uv9nptj31401hctma.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8v09z04j31401404by.jpg\" referrerpolicy=\"no-referrer\"><br><br>";
|
||||
List<String> strings = extractImageUrls(str);
|
||||
strings.forEach(obj->{
|
||||
System.out.println(obj);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static String readHtmlFile(String filePath) throws IOException {
|
||||
StringBuilder content = new StringBuilder();
|
||||
BufferedReader reader = new BufferedReader(new FileReader(filePath));
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
content.append(line);
|
||||
}
|
||||
reader.close();
|
||||
return content.toString();
|
||||
}
|
||||
|
||||
public static List<String> extractImageUrls(String htmlContent) {
|
||||
List<String> imageUrls = new ArrayList<>();
|
||||
String regex = "<img\\s+[^>]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>";
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
Matcher matcher = pattern.matcher(htmlContent);
|
||||
while (matcher.find()) {
|
||||
String imageUrl = matcher.group(1);
|
||||
imageUrls.add(imageUrl);
|
||||
}
|
||||
return imageUrls;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.net.URL;
|
||||
|
||||
/**
|
||||
* RSS获取 测试类
|
||||
*
|
||||
* @author yanqs
|
||||
*/
|
||||
public class RSSReader {
|
||||
public static void main(String[] args) throws Exception {
|
||||
// RSS feed URL
|
||||
// URL rssUrl = new URL("https://rsshub.ys.bcrjl.com/weibo/user/6489032761");
|
||||
URL rssUrl = new URL("https://blog.yanqingshan.com/feed/");
|
||||
|
||||
// Create a DocumentBuilder
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder builder = factory.newDocumentBuilder();
|
||||
|
||||
// Parse the RSS file
|
||||
Document document = builder.parse(rssUrl.openStream());
|
||||
|
||||
// Get all items
|
||||
NodeList items = document.getElementsByTagName("item");
|
||||
|
||||
for (int i = 0; i < items.getLength(); i++) {
|
||||
Element item = (Element) items.item(i);
|
||||
Element title = (Element) item.getElementsByTagName("title").item(0);
|
||||
Element link = (Element) item.getElementsByTagName("link").item(0);
|
||||
|
||||
// Print the title
|
||||
System.out.println(title.getTextContent());
|
||||
System.out.println(link.getTextContent());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package com.bcrjl.rss.common.util;
|
||||
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.http.Header;
|
||||
import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
|
||||
import static com.bcrjl.rss.common.constant.AppConstant.IMAGES_PATH;
|
||||
import static com.bcrjl.rss.common.constant.AppConstant.USER_AGENT;
|
||||
|
||||
/**
|
||||
* Html 工具类测试
|
||||
*
|
||||
* @author yanqs
|
||||
*/
|
||||
class HtmlUtilsTest {
|
||||
|
||||
/**
|
||||
* 获取html中的图片
|
||||
*/
|
||||
@Test
|
||||
void extractImageUrls() {
|
||||
String str = "为什么温泉♨️水那么黄? <img style=\"\" src=\"https://tvax1.sinaimg.cn/large/00759jQJly1hsg8uwgmavj31401hcdy4.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uvtqz7j31401hck91.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax2.sinaimg.cn/large/00759jQJly1hsg8ux1psqj31401z44h1.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8uxycmvj31401hc16o.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax2.sinaimg.cn/large/00759jQJly1hsg8uyn21kj31401hc4gl.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8uzeqdzj31401hck84.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uzyy3yj3140140gz4.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax4.sinaimg.cn/large/00759jQJly1hsg8uv9nptj31401hctma.jpg\" referrerpolicy=\"no-referrer\"><br><br><img style=\"\" src=\"https://tvax3.sinaimg.cn/large/00759jQJly1hsg8v09z04j31401404by.jpg\" referrerpolicy=\"no-referrer\"><br><br>";
|
||||
List<String> strings = HtmlUtils.extractImageUrls(str);
|
||||
strings.forEach(System.out::println);
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试获取文件名
|
||||
*/
|
||||
@Test
|
||||
void getFileName() {
|
||||
String imgSrc = "https://h5.sinaimg.cn/m/emoticon/icon/others/ct_kele-4ce616ef95.png";
|
||||
String videoSrc = "https://f.video.weibocdn.com/o0/H5AopYlvlx08hlyjma2A010412003vHq0E010.mp4?label=mp4_ld&template=360x480.24.0&ori=0&ps=1CwnkDw1GXwCQx&Expires=1724122135&ssig=QYOiNA1GMH&KID=unistore,video";
|
||||
System.out.println(HtmlUtils.getFileName(imgSrc));
|
||||
System.out.println(HtmlUtils.getFileName(videoSrc));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取微博图片流
|
||||
*/
|
||||
@Test
|
||||
void getWeiBoImagesHttpRequest() {
|
||||
String url="https://f.video.weibocdn.com/o0/Eglhl42Glx08hlnyHMZa01041200fr9J0E010.mp4?label=mp4_720p&template=720x1280.24.0&ori=0&ps=1CwnkDw1GXwCQx&Expires=1724122135&ssig=KmPchkXSzw&KID=unistore,video";
|
||||
HttpResponse weiBoVideoHttpRequest = HtmlUtils.getWeiBoVideoHttpRequest(url);
|
||||
byte[] bytes = weiBoVideoHttpRequest.bodyBytes();
|
||||
FileUtil.writeBytes(bytes, new File(IMAGES_PATH + "Eglhl42Glx08hlnyHMZa01041200fr9J0E010.mp4"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取微博视频流
|
||||
*/
|
||||
@Test
|
||||
void getWeiBoVideosHttpRequest() {
|
||||
String url="https://f.video.weibocdn.com/o0/Eglhl42Glx08hlnyHMZa01041200fr9J0E010.mp4?label=mp4_720p&template=720x1280.24.0&ori=0&ps=1CwnkDw1GXwCQx&Expires=1724122135&ssig=KmPchkXSzw&KID=unistore,video";
|
||||
HttpRequest request = HttpRequest.get(url)
|
||||
.header(Header.REFERER, "https://weibo.com/")
|
||||
.header(Header.USER_AGENT, USER_AGENT)
|
||||
.timeout(20000);
|
||||
HttpResponse httpResponse = request.executeAsync();
|
||||
byte[] bytes = httpResponse.bodyBytes();
|
||||
FileUtil.writeBytes(bytes, new File(IMAGES_PATH + "Eglhl42Glx08hlnyHMZa01041200fr9J0E010.mp4"));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +1,21 @@
|
|||
package com.bcrjl.rss.common.util;
|
||||
|
||||
import cn.hutool.core.lang.Console;
|
||||
import cn.hutool.extra.mail.MailAccount;
|
||||
import com.bcrjl.rss.common.util.MailUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* 邮件测试
|
||||
*
|
||||
* @author yanqs
|
||||
* @since 2024-08-09
|
||||
*/
|
||||
public class MailTest {
|
||||
public static void main(String[] args) {
|
||||
class MailUtilsTest {
|
||||
|
||||
/**
|
||||
* 初始化邮件账号
|
||||
*/
|
||||
@Test
|
||||
void initMailAccount() {
|
||||
MailAccount mailAccount = MailUtils.initMailAccount();
|
||||
Console.log(mailAccount);
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package com.bcrjl.rss.common.util;
|
||||
|
||||
import cn.hutool.core.lang.Console;
|
||||
import com.bcrjl.rss.model.entity.RssEntity;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* RSSUtilsTest
|
||||
*
|
||||
* @author yanqs
|
||||
*/
|
||||
class RssUtilsTest {
|
||||
|
||||
@Test
|
||||
void getRssList() {
|
||||
String url = "https://blog.yanqingshan.com/feed/";
|
||||
List<RssEntity> rssList = RssUtils.getRssList(url);
|
||||
rssList.forEach(obj -> {
|
||||
Console.log(obj);
|
||||
});
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue