From f1f75bc4666213264cce62cfd7cf36c28d47512e Mon Sep 17 00:00:00 2001 From: yanqs Date: Wed, 14 Aug 2024 15:37:33 +0800 Subject: [PATCH] init:0.4.0 --- Dockerfile | 20 ++ config/config.setting | 32 +++ config/data.json | 11 + config/rss-data.json | 1 + docker/docker-compose.yaml | 15 ++ pom.xml | 174 +++++++++++++ .../java/com/bcrjl/rss/WebApplication.java | 21 ++ .../java/com/bcrjl/rss/cache/ConfigCache.java | 25 ++ .../rss/common/constant/AppConstant.java | 65 +++++ .../com/bcrjl/rss/common/util/AListUtils.java | 95 +++++++ .../bcrjl/rss/common/util/HtmlParseUtils.java | 59 +++++ .../com/bcrjl/rss/common/util/MailUtils.java | 37 +++ .../com/bcrjl/rss/common/util/RssUtils.java | 63 +++++ .../rss/config/DynamicScheduleConfig.java | 59 +++++ .../bcrjl/rss/config/SaTokenConfigure.java | 24 ++ .../com/bcrjl/rss/config/WebConfigurer.java | 44 ++++ .../bcrjl/rss/controller/RssController.java | 19 ++ .../rss/controller/SystemController.java | 27 ++ .../java/com/bcrjl/rss/job/FileMonitor.java | 111 ++++++++ src/main/java/com/bcrjl/rss/job/RssJob.java | 141 ++++++++++ .../bcrjl/rss/model/entity/DataEntity.java | 27 ++ .../com/bcrjl/rss/model/entity/RssEntity.java | 37 +++ src/main/resources/application-rss.yml | 0 src/main/resources/application.yml | 54 ++++ src/main/resources/banner.txt | 10 + src/main/resources/logback-spring.xml | 240 ++++++++++++++++++ src/test/java/HtmlTest.java | 51 ++++ src/test/java/MailTest.java | 14 + src/test/java/RSSReader.java | 40 +++ 29 files changed, 1516 insertions(+) create mode 100644 Dockerfile create mode 100644 config/config.setting create mode 100644 config/data.json create mode 100644 config/rss-data.json create mode 100644 docker/docker-compose.yaml create mode 100644 pom.xml create mode 100644 src/main/java/com/bcrjl/rss/WebApplication.java create mode 100644 src/main/java/com/bcrjl/rss/cache/ConfigCache.java create mode 100644 src/main/java/com/bcrjl/rss/common/constant/AppConstant.java create mode 100644 src/main/java/com/bcrjl/rss/common/util/AListUtils.java create mode 100644 src/main/java/com/bcrjl/rss/common/util/HtmlParseUtils.java create mode 100644 src/main/java/com/bcrjl/rss/common/util/MailUtils.java create mode 100644 src/main/java/com/bcrjl/rss/common/util/RssUtils.java create mode 100644 src/main/java/com/bcrjl/rss/config/DynamicScheduleConfig.java create mode 100644 src/main/java/com/bcrjl/rss/config/SaTokenConfigure.java create mode 100644 src/main/java/com/bcrjl/rss/config/WebConfigurer.java create mode 100644 src/main/java/com/bcrjl/rss/controller/RssController.java create mode 100644 src/main/java/com/bcrjl/rss/controller/SystemController.java create mode 100644 src/main/java/com/bcrjl/rss/job/FileMonitor.java create mode 100644 src/main/java/com/bcrjl/rss/job/RssJob.java create mode 100644 src/main/java/com/bcrjl/rss/model/entity/DataEntity.java create mode 100644 src/main/java/com/bcrjl/rss/model/entity/RssEntity.java create mode 100644 src/main/resources/application-rss.yml create mode 100644 src/main/resources/application.yml create mode 100644 src/main/resources/banner.txt create mode 100644 src/main/resources/logback-spring.xml create mode 100644 src/test/java/HtmlTest.java create mode 100644 src/test/java/MailTest.java create mode 100644 src/test/java/RSSReader.java diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f2d2dfd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# 基于java镜像创建新镜像 +FROM adoptopenjdk/openjdk8-openj9:jdk8u412-b08_openj9-0.44.0-alpine-slim +# 作者 +MAINTAINER bcrjl + +EXPOSE 24803 + +WORKDIR /app + +# 创建 bin 目录 +RUN mkdir -p bin \ + && mkdir -p config \ + && mkdir -p lib \ + && mkdir -p log + +COPY ./target/rss-reader.jar /app/rss-reader.jar +COPY ./target/lib/* /app/lib/ +COPY ./config/* /app/config/ + +ENTRYPOINT ["java","-Dloader.path=/app/lib/", "-Djava.security.egd=file:/dev/./urandom", "-jar", "/app/rss-reader.jar"] diff --git a/config/config.setting b/config/config.setting new file mode 100644 index 0000000..4d573c9 --- /dev/null +++ b/config/config.setting @@ -0,0 +1,32 @@ +[system] +## 配置订阅频率 +refresh=5 +## 保存微博图片 +saveWeiBoImages=true +## 上传图片到AList +uploadAList=false +## AList Url +aListUrl= +## AList 账号 +aListUser= +## AList 密码 +aListPass= +## AList 上传路径 +aListUploadPath= +[mail] +## 启用邮件推送 +enable=false +## 启用邮件更新推送 +sendUpdate=true +## 邮件服务器的SMTP地址，可选，默认为smtp.<发件人邮箱后缀> +host=smtp.qq.com +## 邮件服务器的SMTP端口，可选，默认25 +port=465 +## 发件人（必须正确，否则发送失败） +from=rss-reply +## 用户名，默认为发件人邮箱前缀 +user=xxxxxxxxxxxxxx +## 密码（注意，某些邮箱需要为SMTP服务单独设置授权码，详情查看相关帮助） +pass= +## 接收人，多人以英文逗号分割 +to= diff --git a/config/data.json b/config/data.json new file mode 100644 index 0000000..7689c4a --- /dev/null +++ b/config/data.json @@ -0,0 +1,11 @@ +{ + "rssList": { + "weibo": [ + + ], + "other": [ + + ] + } +} + diff --git a/config/rss-data.json b/config/rss-data.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/config/rss-data.json @@ -0,0 +1 @@ +[] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..c7cbce1 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,15 @@ +version: "2.24.6" +services: + rss-reader: + image: bcrjl/rss-reader:latest + container_name: rss-reader + restart: always + volumes: + - ./config:/app/config + - ./log:/app/log + - ./images:/app/images + - /etc/localtime:/etc/localtime:ro + environment: + - TZ=Asia/Shanghai + ports: + - '24803:24803' diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..bf18ea8 --- /dev/null +++ b/pom.xml @@ -0,0 +1,174 @@ + + + 4.0.0 + + com.bcrjl.rss + rss-reader + 0.4.0 + RSS订阅阅读器 + + + 8 + 8 + 3.8.1 + 3.3.0 + 3.2.0 + 3.2.0 + 1.8 + UTF-8 + + 2.7.18 + 1.38.0 + 1.18.34 + 5.8.29 + 1.6.2 + + + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring.boot.version} + pom + import + + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-thymeleaf + + + + cn.dev33 + sa-token-spring-boot-starter + ${sa-token.version} + + + + cn.dev33 + sa-token-jwt + ${sa-token.version} + + + cn.hutool + hutool-jwt + + + + + + commons-io + commons-io + 2.11.0 + + + + + cn.hutool + hutool-all + ${hutool.version} + + + + + com.sun.mail + javax.mail + ${javax.mail.version} + + + + + + org.projectlombok + lombok + ${lombok.version} + compile + + + + + + + ${project.artifactId} + + + org.springframework.boot + spring-boot-maven-plugin + ${spring.boot.version} + + true + ZIP + + + + nothing + nothing + + + + + + + repackage + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy + package + + copy-dependencies + + + + + ${project.build.directory}/lib + + + + + + + + + + + + + nexus + nexus-developer + https://nexus.ys.bcrjl.com:41010/repository/maven-public/ + + true + + + + + + nexus + https://nexus.ys.bcrjl.com:41010/repository/maven-public/ + + + + diff --git a/src/main/java/com/bcrjl/rss/WebApplication.java b/src/main/java/com/bcrjl/rss/WebApplication.java new file mode 100644 index 0000000..fe970c3 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/WebApplication.java @@ -0,0 +1,21 @@ +package com.bcrjl.rss; + +import com.bcrjl.rss.job.FileMonitor; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +import javax.annotation.Resource; + +/** + * 项目启动类 + * + * @author yanqs + */ +@EnableScheduling +@SpringBootApplication +public class WebApplication { + public static void main(String[] args) { + SpringApplication.run(WebApplication.class, args); + } +} diff --git a/src/main/java/com/bcrjl/rss/cache/ConfigCache.java b/src/main/java/com/bcrjl/rss/cache/ConfigCache.java new file mode 100644 index 0000000..10d6a41 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/cache/ConfigCache.java @@ -0,0 +1,25 @@ +package com.bcrjl.rss.cache; + +import cn.hutool.cache.Cache; +import cn.hutool.cache.CacheUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +/** + * 缓存配置 + * + * @author yanqs + */ +@Slf4j +@Component +public class ConfigCache { + public static Cache fifoCache = CacheUtil.newFIFOCache(10); + + public static Object getConfig(String key) { + return fifoCache.get(key); + } + + public static void setConfig(String key, Object value) { + fifoCache.put(key, value); + } +} diff --git a/src/main/java/com/bcrjl/rss/common/constant/AppConstant.java b/src/main/java/com/bcrjl/rss/common/constant/AppConstant.java new file mode 100644 index 0000000..ee3bcac --- /dev/null +++ b/src/main/java/com/bcrjl/rss/common/constant/AppConstant.java @@ -0,0 +1,65 @@ +package com.bcrjl.rss.common.constant; + +/** + * 应用常量 + * + * @author yanqs + */ +public interface AppConstant { + + int INIT_MAP = 16; + + /** + * 系统配置路径 + */ + String CONFIG_PATH = System.getProperty("user.dir") + "/config/config.setting"; + /** + * RSS数据路径 + */ + String RSS_DATA_PATH = System.getProperty("user.dir") + "/config/rss-data.json"; + + /** + * RSS订阅源路径 + */ + String RSS_CONFIG_PATH = System.getProperty("user.dir") + "/config/data.json"; + String IMAGES_PATH = System.getProperty("user.dir") + "/images/"; + + String SET_SYSTEM = "system"; + String SET_MAIL = "mail"; + /** + * 配置RSS订阅频率 + */ + String REFRESH = "refresh"; + + /** + * 保存微博图片 + */ + String SAVE_WEIBO_IMAGES = "saveWeiBoImages"; + + /** + * 上传图片到AList + */ + String UPLOAD_ALIST = "uploadAList"; + + /** + * AList Url + */ + String ALIST_URL = "aListUrl"; + /** + * AList 账号 + */ + String ALIST_USER = "aListUser"; + /** + * AList 密码 + */ + String ALIST_PASS = "aListPass"; + + String ALIST_UPLOAD_PATH = "aListUploadPath"; + + Integer MAIL_CONFIG_SIZE = 7; + + String MAIL_CONFIG_ENABLE = "enable"; + + String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"; + +} diff --git a/src/main/java/com/bcrjl/rss/common/util/AListUtils.java b/src/main/java/com/bcrjl/rss/common/util/AListUtils.java new file mode 100644 index 0000000..9a576b5 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/common/util/AListUtils.java @@ -0,0 +1,95 @@ +package com.bcrjl.rss.common.util; + +import cn.hutool.cache.CacheUtil; +import cn.hutool.cache.impl.TimedCache; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.CharsetUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.http.ContentType; +import cn.hutool.http.Header; +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import cn.hutool.json.JSONUtil; +import cn.hutool.setting.Setting; +import lombok.extern.slf4j.Slf4j; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import static com.bcrjl.rss.common.constant.AppConstant.*; + +/** + * 功能描述: + * + * @author yanqs + * @since 2024-08-10 + */ +@Slf4j +public class AListUtils { + + private static final String TOKEN_URL = "/api/auth/login"; + + private static final String UPLOAD_URL = "/api/fs/put"; + + private static TimedCache timedCache = CacheUtil.newTimedCache(86400000); + + /** + * 获取AList Token + * + * @return token + */ + private static String getToken() { + try { + String alistToken = timedCache.get("alist_token"); + if (StrUtil.isNotEmpty(alistToken)) { + return alistToken; + } else { + Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true); + Setting systemSetting = setting.getSetting(SET_SYSTEM); + Map params = new HashMap<>(INIT_MAP); + params.put("username", systemSetting.get(ALIST_USER)); + params.put("password", systemSetting.get(ALIST_PASS)); + String aListUrl = systemSetting.get(ALIST_URL); + String body = HttpRequest.post(aListUrl + TOKEN_URL) + .header(Header.CONTENT_TYPE, ContentType.JSON.getValue()) + .body(JSONUtil.toJsonStr(params)) + .execute().body(); + String token = JSONUtil.parseObj(body).getJSONObject("data").getStr("token"); + timedCache.put("alist_token", token); + return token; + } + } catch (Exception e) { + log.error("获取AList Token异常：", e); + return ""; + } + } + + public static void uploadFile(byte[] fileByte, String fileName) { + try { + String time = DateUtil.format(new Date(), "yyyyMMddHH"); + Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true); + Setting systemSetting = setting.getSetting(SET_SYSTEM); + String aListUrl = systemSetting.get(ALIST_URL); + String uploadPath = systemSetting.get(ALIST_UPLOAD_PATH); + HttpResponse httpResponse = HttpRequest.put(aListUrl + UPLOAD_URL) + .header(Header.AUTHORIZATION, getToken()) + .header(Header.CONTENT_TYPE, ContentType.MULTIPART.getValue()) + .header("File-Path", uploadPath + "/" + time + "/" + fileName) + .body(fileByte) + .execute(); + if (httpResponse.isOk()) { + String message = JSONUtil.parseObj(httpResponse.body()).getStr("message"); + if ("success".equals(message)) { + //log.info("AList上传成功"); + } else { + log.info("AList上传失败：{}", message); + } + } else { + log.info("AList上传失败,响应状态码:{}", httpResponse.getStatus()); + } + } catch (Exception e) { + log.error("AList上传文件异常：", e); + } + } +} diff --git a/src/main/java/com/bcrjl/rss/common/util/HtmlParseUtils.java b/src/main/java/com/bcrjl/rss/common/util/HtmlParseUtils.java new file mode 100644 index 0000000..08ab846 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/common/util/HtmlParseUtils.java @@ -0,0 +1,59 @@ +package com.bcrjl.rss.common.util; + +import cn.hutool.http.Header; +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.bcrjl.rss.common.constant.AppConstant.USER_AGENT; + +/** + * Html 工具类 + * + * @author yanqs + */ +@Slf4j +public class HtmlParseUtils { + /** + * 获取html中的图片 + * + * @param htmlContent html内容 + * @return + */ + public static List extractImageUrls(String htmlContent) { + List imageUrls = new ArrayList<>(); + String regex = "]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(htmlContent); + while (matcher.find()) { + String imageUrl = matcher.group(1); + imageUrls.add(imageUrl); + } + return imageUrls; + } + + /** + * 获取微博图片流文件 + * + * @param fileName 微博图片名称 + * @return HttpResponse + */ + public static HttpResponse getWeiBoImagesHttpRequest(String fileName) { + try { + String url = "https://tvax3.sinaimg.cn/large/" + fileName; + HttpRequest request = HttpRequest.get(url) + .header(Header.REFERER, "https://weibo.com/") + .header(Header.USER_AGENT, USER_AGENT) + .timeout(20000); + return request.executeAsync(); + } catch (Exception e) { + log.error("获取微博图片数据异常：", e); + return null; + } + } +} diff --git a/src/main/java/com/bcrjl/rss/common/util/MailUtils.java b/src/main/java/com/bcrjl/rss/common/util/MailUtils.java new file mode 100644 index 0000000..ed71a2c --- /dev/null +++ b/src/main/java/com/bcrjl/rss/common/util/MailUtils.java @@ -0,0 +1,37 @@ +package com.bcrjl.rss.common.util; + +import cn.hutool.core.util.CharsetUtil; +import cn.hutool.extra.mail.MailAccount; +import cn.hutool.setting.Setting; +import lombok.extern.slf4j.Slf4j; + +import static com.bcrjl.rss.common.constant.AppConstant.*; + +/** + * 邮件工具 + * + * @author yanqs + * @since 2024-08-07 + */ +@Slf4j +public class MailUtils { + public static MailAccount initMailAccount() { + Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true); + Setting emailSetting = setting.getSetting(SET_MAIL); + if (emailSetting.size() >= MAIL_CONFIG_SIZE) { + MailAccount mailAccount = new MailAccount(); + mailAccount.setHost(emailSetting.getWithLog("host")); + mailAccount.setPort(Integer.valueOf(emailSetting.getWithLog("port"))); + mailAccount.setAuth(true); + mailAccount.setFrom(emailSetting.getWithLog("from")); + mailAccount.setUser(emailSetting.getWithLog("user")); + mailAccount.setPass(emailSetting.getWithLog("pass")); + mailAccount.setSslEnable(true); + mailAccount.setDebug(false); + return mailAccount; + } else { + log.error("请配置邮箱信息"); + return null; + } + } +} diff --git a/src/main/java/com/bcrjl/rss/common/util/RssUtils.java b/src/main/java/com/bcrjl/rss/common/util/RssUtils.java new file mode 100644 index 0000000..ecc1930 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/common/util/RssUtils.java @@ -0,0 +1,63 @@ +package com.bcrjl.rss.common.util; + +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.bcrjl.rss.model.entity.RssEntity; +import lombok.extern.slf4j.Slf4j; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +/** + * RSS 订阅工具类 + * + * @author yanqs + */ +@Slf4j +public class RssUtils { + + public static List getRssList(String url) { + List rssList = new ArrayList<>(); + log.info("开始订阅：{}", url); + try { + //URL rssUrl = new URL(url); + HttpResponse httpResponse = HttpRequest.get(url).executeAsync(); + if(httpResponse.isOk()){ + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(httpResponse.bodyStream()); + NodeList items = document.getElementsByTagName("item"); + Node webTitle = document.getElementsByTagName("title").item(0); + for (int i = 0; i < items.getLength(); i++) { + Element item = (Element) items.item(i); + Element title = (Element) item.getElementsByTagName("title").item(0); + Element link = (Element) item.getElementsByTagName("link").item(0); + Element description = (Element) item.getElementsByTagName("description").item(0); + Element pubDate = (Element) item.getElementsByTagName("pubDate").item(0); + rssList.add(RssEntity.builder() + .webTitle(webTitle.getTextContent()) + .url(url) + .title(title.getTextContent()) + .link(link.getTextContent()) + .description(description.getTextContent()) + .createTime(DateUtil.parse(pubDate.getTextContent(), DatePattern.HTTP_DATETIME_FORMAT)) + .build()); + } + } + + + } catch (Exception e) { + log.error("地址：{}获取RSS订阅内容异常：", url, e); + } + return rssList; + } +} diff --git a/src/main/java/com/bcrjl/rss/config/DynamicScheduleConfig.java b/src/main/java/com/bcrjl/rss/config/DynamicScheduleConfig.java new file mode 100644 index 0000000..e0b256d --- /dev/null +++ b/src/main/java/com/bcrjl/rss/config/DynamicScheduleConfig.java @@ -0,0 +1,59 @@ +package com.bcrjl.rss.config; + +import cn.hutool.core.util.CharsetUtil; +import cn.hutool.setting.Setting; +import com.bcrjl.rss.job.RssJob; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.Trigger; +import org.springframework.scheduling.TriggerContext; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; +import org.springframework.scheduling.support.PeriodicTrigger; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.Objects; + +import static com.bcrjl.rss.common.constant.AppConstant.REFRESH; + +/** + * 动态配置订阅 + * + * @author yanqs + * @since 2024-08-06 + */ +@Data +@Slf4j +@Configuration +@EnableScheduling +public class DynamicScheduleConfig implements SchedulingConfigurer { + + @Resource + private RssJob rssJob; + + /** + * 默认五分钟执行一次 + */ + private Long timer = 5L * 1000L * 60L; + + @Override + public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { + taskRegistrar.addTriggerTask(new Runnable() { + @Override + public void run() { + rssJob.subscribe(); + } + }, new Trigger() { + @Override + public Date nextExecutionTime(TriggerContext triggerContext) { + log.info("当前执行速度：{}分钟", timer / 1000L / 60L); + PeriodicTrigger periodicTrigger = new PeriodicTrigger(timer); + Date nextExecutionTime = periodicTrigger.nextExecutionTime(triggerContext); + return nextExecutionTime; + } + }); + } +} diff --git a/src/main/java/com/bcrjl/rss/config/SaTokenConfigure.java b/src/main/java/com/bcrjl/rss/config/SaTokenConfigure.java new file mode 100644 index 0000000..ccd0f06 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/config/SaTokenConfigure.java @@ -0,0 +1,24 @@ +package com.bcrjl.rss.config; + +import cn.dev33.satoken.jwt.StpLogicJwtForSimple; +import cn.dev33.satoken.stp.StpLogic; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Sa-Token 配置 + * + * @author yanqs + */ +@Configuration +public class SaTokenConfigure { + /** + * Sa-Token 整合 jwt (Simple 简单模式) + * + * @return + */ + @Bean + public StpLogic getStpLogicJwt() { + return new StpLogicJwtForSimple(); + } +} diff --git a/src/main/java/com/bcrjl/rss/config/WebConfigurer.java b/src/main/java/com/bcrjl/rss/config/WebConfigurer.java new file mode 100644 index 0000000..a4e788f --- /dev/null +++ b/src/main/java/com/bcrjl/rss/config/WebConfigurer.java @@ -0,0 +1,44 @@ +package com.bcrjl.rss.config; + +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.UrlBasedCorsConfigurationSource; +import org.springframework.web.filter.CorsFilter; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurationSupport; + +/** + * @author yanqs + * @since 2024-08-03 + */ +@Configuration +public class WebConfigurer extends WebMvcConfigurationSupport { + /** + * 跨域过滤 + * + * @return FilterRegistrationBean + */ + @Bean + public FilterRegistrationBean corsFilter() { + UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); + source.registerCorsConfiguration("/**", corsConfig()); + FilterRegistrationBean bean = new FilterRegistrationBean(new CorsFilter(source)); + //*****这里设置了优先级***** + bean.setOrder(1); + return bean; + } + + private CorsConfiguration corsConfig() { + CorsConfiguration corsConfiguration = new CorsConfiguration(); + //corsConfiguration.addAllowedOrigin("*"); + corsConfiguration.addAllowedOriginPattern("*"); + corsConfiguration.addAllowedHeader("*"); + corsConfiguration.addAllowedMethod("*"); + corsConfiguration.setAllowCredentials(true); + corsConfiguration.setMaxAge(3600L); + return corsConfiguration; + } + +} diff --git a/src/main/java/com/bcrjl/rss/controller/RssController.java b/src/main/java/com/bcrjl/rss/controller/RssController.java new file mode 100644 index 0000000..cc1915d --- /dev/null +++ b/src/main/java/com/bcrjl/rss/controller/RssController.java @@ -0,0 +1,19 @@ +package com.bcrjl.rss.controller; + +import com.bcrjl.rss.common.util.RssUtils; +import lombok.AllArgsConstructor; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * @author yanqs + * @since 2024-08-03 + */ +@AllArgsConstructor +@RestController +@RequestMapping("/rss") +public class RssController { + +} diff --git a/src/main/java/com/bcrjl/rss/controller/SystemController.java b/src/main/java/com/bcrjl/rss/controller/SystemController.java new file mode 100644 index 0000000..f053ff3 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/controller/SystemController.java @@ -0,0 +1,27 @@ +package com.bcrjl.rss.controller; + +import cn.hutool.core.io.resource.ResourceUtil; +import cn.hutool.core.util.CharsetUtil; +import cn.hutool.setting.Setting; +import lombok.AllArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import static com.bcrjl.rss.common.constant.AppConstant.CONFIG_PATH; + +/** + * @author yanqs + * @since 2024-08-07 + */ +@AllArgsConstructor +@RestController +@RequestMapping("/system") +public class SystemController { + + @GetMapping("/config") + public Object getConfig(){ + String str = ResourceUtil.readUtf8Str(CONFIG_PATH); + return str; + } +} diff --git a/src/main/java/com/bcrjl/rss/job/FileMonitor.java b/src/main/java/com/bcrjl/rss/job/FileMonitor.java new file mode 100644 index 0000000..ef64168 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/job/FileMonitor.java @@ -0,0 +1,111 @@ +package com.bcrjl.rss.job; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.io.file.FileWriter; +import cn.hutool.core.util.CharsetUtil; +import cn.hutool.json.JSONUtil; +import cn.hutool.setting.Setting; +import com.bcrjl.rss.cache.ConfigCache; +import com.bcrjl.rss.config.DynamicScheduleConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static com.bcrjl.rss.common.constant.AppConstant.*; + +/** + * 监控配置文件 + * + * @author yanqs + * @since 2024-08-06 + */ +@Slf4j +@Component +public class FileMonitor { + + @Resource + private DynamicScheduleConfig scheduledTask; + + @PostConstruct + public void initConfig() { + existFile("config.setting", "[system]\n" + + "## 配置订阅频率\n" + + "refresh = 5\n" + + "## 保存微博图片\n" + + "saveWeiBoImages=false\n" + + "## 上传图片到AList\n" + + "uploadAList=false\n" + + "## AList Url\n" + + "aListUrl=\n" + + "## AList 账号\n" + + "aListUser=\n" + + "## AList 密码\n" + + "aListPass=\n" + + "## AList 上传路径\n" + + "aListUploadPath=" + + "[mail]\n" + + "## 启用邮件推送\n" + + "enable=false\n" + + "## 启用邮件更新推送\n" + + "sendUpdate=true\n" + + "## 邮件服务器的SMTP地址，可选，默认为smtp.<发件人邮箱后缀>\n" + + "host=smtp.qq.com\n" + + "## 邮件服务器的SMTP端口，可选，默认25\n" + + "port=465\n" + + "## 发件人（必须正确，否则发送失败）\n" + + "from=rss-reply\n" + + "## 用户名，默认为发件人邮箱前缀\n" + + "user=xxxxxxxxxxxxxx\n" + + "## 密码（注意，某些邮箱需要为SMTP服务单独设置授权码，详情查看相关帮助）\n" + + "pass=\n" + + "## 接收人，多人以英文逗号分割\n" + + "to=\n"); + Map dataJson = new HashMap<>(); + Map rssList = new HashMap<>(); + rssList.put("weibo",new ArrayList<>()); + rssList.put("other",new ArrayList<>()); + dataJson.put("rssList",rssList); + existFile("data.json", JSONUtil.toJsonPrettyStr(dataJson)); + existFile("rss-data.json", "[]"); + + } + + private void existFile(String fileName, String fileContent) { + fileName = System.getProperty("user.dir") + "/config/" + fileName; + File file = FileUtil.file(fileName); + if (!FileUtil.exist(file)) { + file = FileUtil.touch(fileName); + FileWriter dataFile = new FileWriter(file); + dataFile.write(fileContent, false); + log.info("监听配置文件不存在，系统已自动创建"); + } + } + + /** + * 文件监听 + */ + @Scheduled(cron = "0/10 * * * * ?") + public void configMonitor() { + Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true); + Long refresh = Long.valueOf(setting.getByGroup(REFRESH, SET_SYSTEM)); + Long config = (Long) ConfigCache.getConfig(REFRESH); + if (Objects.isNull(config)) { + ConfigCache.setConfig(REFRESH, refresh); + config = refresh; + } + if (!config.equals(refresh)) { + log.info("配置变动，更新频率：{}分钟", refresh); + scheduledTask.setTimer(refresh * 60 * 1000L); + ConfigCache.setConfig(REFRESH, refresh); + } + } +} + diff --git a/src/main/java/com/bcrjl/rss/job/RssJob.java b/src/main/java/com/bcrjl/rss/job/RssJob.java new file mode 100644 index 0000000..e784c6f --- /dev/null +++ b/src/main/java/com/bcrjl/rss/job/RssJob.java @@ -0,0 +1,141 @@ +package com.bcrjl.rss.job; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.io.file.FileWriter; +import cn.hutool.core.io.resource.ResourceUtil; +import cn.hutool.core.util.CharsetUtil; +import cn.hutool.extra.mail.MailAccount; +import cn.hutool.extra.mail.MailUtil; +import cn.hutool.http.HttpResponse; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import cn.hutool.setting.Setting; +import com.bcrjl.rss.common.util.AListUtils; +import com.bcrjl.rss.common.util.HtmlParseUtils; +import com.bcrjl.rss.common.util.MailUtils; +import com.bcrjl.rss.common.util.RssUtils; +import com.bcrjl.rss.model.entity.RssEntity; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static com.bcrjl.rss.common.constant.AppConstant.*; + +/** + * RSS 任务 + * + * @author yanqs + */ +@Slf4j +@Component +public class RssJob { + + /** + * RSS 订阅 + */ + public void subscribe() { + if (!FileUtil.exist(RSS_DATA_PATH)) { + FileUtil.touch(RSS_DATA_PATH); + FileWriter dataFile = new FileWriter(RSS_DATA_PATH); + dataFile.write("[]"); + } + JSONObject rssListObj = JSONUtil.parseObj(ResourceUtil.readUtf8Str(RSS_CONFIG_PATH)).getJSONObject("rssList"); + if (Objects.nonNull(rssListObj)) { + List dbList = JSONUtil.toList(ResourceUtil.readUtf8Str(RSS_DATA_PATH), RssEntity.class); + List weiboRssList = JSONUtil.toList(rssListObj.getStr("weibo"), String.class); + List saveList = new ArrayList<>(); + weiboRssList.forEach(obj -> { + List rssList = RssUtils.getRssList(obj); + saveList.addAll(rssList); + }); + List otherRssList = JSONUtil.toList(rssListObj.getStr("other"), String.class); + otherRssList.forEach(obj -> { + List rssList = RssUtils.getRssList(obj); + saveList.addAll(rssList); + }); + if (CollUtil.isEmpty(dbList)) { + FileWriter dataFile = new FileWriter(RSS_DATA_PATH); + dataFile.write(JSONUtil.toJsonPrettyStr(saveList)); + sendEmailReply(saveList); + } else { + List dbIds = dbList.stream().map(RssEntity::getLink).collect(Collectors.toList()); + // 获取库中不包含的数据 则为新增数据 + List insertList = saveList.stream() + .filter(t -> !dbIds.contains(t.getLink())).collect(Collectors.toList()); + dbList.addAll(insertList); + FileWriter dataFile = new FileWriter(RSS_DATA_PATH); + dataFile.write(JSONUtil.toJsonPrettyStr(dbList)); + sendEmailReply(insertList); + log.info("本次新增了{}条推送内容", insertList.size()); + } + } else { + log.info("未配置订阅信息"); + } + log.info("订阅数据结束！"); + } + + /** + * 发送邮件通知 + * + * @param list rssList + */ + private void sendEmailReply(List list) { + if (CollUtil.isNotEmpty(list)) { + Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true); + Setting emailSetting = setting.getSetting(SET_MAIL); + saveWeiBoImagesOrUpdateAlist(list); + if (emailSetting.getBool(MAIL_CONFIG_ENABLE) && emailSetting.getBool("sendUpdate")) { + // 如果邮箱开启且发送更新邮件开启 则推送通知 + StringBuffer stringBuffer = new StringBuffer(); + list.forEach(obj -> { + stringBuffer.append("
"); + stringBuffer.append("").append(obj.getWebTitle()).append("--").append(obj.getTitle()).append("
"); + stringBuffer.append("
"); + }); + MailAccount mailAccount = MailUtils.initMailAccount(); + List toList = Arrays.asList(emailSetting.getStr("to").split(",")); + if (CollUtil.isNotEmpty(toList)) { + MailUtil.send(mailAccount, toList, "RSS订阅推送", stringBuffer.toString(), true); + } + } + } + } + + /** + * 保存微博图片到本地且上传AList + */ + private void saveWeiBoImagesOrUpdateAlist(List list) { + Setting setting = new Setting(CONFIG_PATH, CharsetUtil.CHARSET_UTF_8, true); + Setting systemSetting = setting.getSetting(SET_SYSTEM); + Boolean saveImages = Boolean.valueOf(systemSetting.get(SAVE_WEIBO_IMAGES)); + Boolean uploadAList = Boolean.valueOf(systemSetting.get(UPLOAD_ALIST)); + if (saveImages) { + // 保存图片 + list.forEach(obj -> { + List imgList = HtmlParseUtils.extractImageUrls(obj.getDescription()); + imgList.forEach(imgObj -> { + if (imgObj.contains("sinaimg") && !imgObj.contains("timeline_card") && !imgObj.contains("qixi2018")) { + int lastSlashIndex = imgObj.lastIndexOf('/'); + // 如果找到了斜杠，就从斜杠后面截取字符串 + String fileName = imgObj.substring(lastSlashIndex + 1); + //log.info("微博图片文件名：{}", fileName); + HttpResponse weiBoImagesHttpRequest = HtmlParseUtils.getWeiBoImagesHttpRequest(fileName); + byte[] bytes = weiBoImagesHttpRequest.bodyBytes(); + FileUtil.writeBytes(bytes, new File(IMAGES_PATH + fileName)); + if (uploadAList) { + AListUtils.uploadFile(bytes, fileName); + } + } + }); + }); + } + } + +} diff --git a/src/main/java/com/bcrjl/rss/model/entity/DataEntity.java b/src/main/java/com/bcrjl/rss/model/entity/DataEntity.java new file mode 100644 index 0000000..f05a8f8 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/model/entity/DataEntity.java @@ -0,0 +1,27 @@ +package com.bcrjl.rss.model.entity; + +import lombok.Data; + +import java.util.List; + +/** + * @author yanqs + * @since 2024-08-03 + */ +@Data +public class DataEntity { + /** + * rss订阅链接（必填） + */ + private RssListEntity rssList; + + /** + * rss订阅更新时间间隔，单位分钟（必填） + */ + private Long refresh; +} + +class RssListEntity { + private List weibo; + private List other; +} diff --git a/src/main/java/com/bcrjl/rss/model/entity/RssEntity.java b/src/main/java/com/bcrjl/rss/model/entity/RssEntity.java new file mode 100644 index 0000000..b462ba6 --- /dev/null +++ b/src/main/java/com/bcrjl/rss/model/entity/RssEntity.java @@ -0,0 +1,37 @@ +package com.bcrjl.rss.model.entity; + +import lombok.Builder; +import lombok.Data; + +import java.util.Date; + +/** + * @author yanqs + * @since 2024-08-03 + */ +@Data +@Builder +public class RssEntity { + /** + * 订阅地址 + */ + private String url; + /** + * 源名称 + */ + private String webTitle; + /** + * 标题 + */ + private String title; + /** + * 链接 + */ + private String link; + /** + * 描述 + */ + private String description; + + private Date createTime; +} diff --git a/src/main/resources/application-rss.yml b/src/main/resources/application-rss.yml new file mode 100644 index 0000000..e69de29 diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..9128431 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,54 @@ +server: + # 服务端口 + port: 24803 + # 是否启用响应压缩，默认为 false + compression: + enabled: true + # 启用压缩的最小响应大小（字节），默认为 2048 + min-response-size: 1024 + tomcat: + # 最大连接数，默认为 10000 + max-connections: 5000 + # 最大线程数，默认为 200 + max-threads: 100 + # 接受队列长度，默认为 100 + accept-count: 200 + # 连接超时时间（毫秒），默认为 20000 + connection-timeout: 30000 + # 长连接的空闲超时时间（毫秒），默认为 20000 + keep-alive-timeout: 30000 + # 最大允许上传的文件大小（字节），默认为 2MB + max-swallow-size: 10485760 + # 最大允许的单个文件大小（字节），默认为 1MB + max-file-size: 5242880 + # 最大允许的请求大小（字节），默认为 10MB + max-request-size: 10485760 + +spring: + application: + name: rss-backend + profiles: + active: rss + web: + resources: + static-locations: classpath:/static/ + ############## Sa-Token 配置 (文档: https://sa-token.cc) ############## +sa-token: + # token 名称（同时也是 cookie 名称） + token-name: Authorization + # token 有效期（单位：秒） 默认30天，-1 代表永久有效 + timeout: 2592000 + # token 最低活跃频率（单位：秒），如果 token 超过此时间没有访问系统就会被冻结，默认-1 代表不限制，永不冻结 + active-timeout: -1 + # 是否允许同一账号多地同时登录 （为 true 时允许一起登录, 为 false 时新登录挤掉旧登录） + is-concurrent: false + # 在多人登录同一账号时，是否共用一个 token （为 true 时所有登录共用一个 token, 为 false 时每次登录新建一个 token） + is-share: true + # token 风格（默认可取值：uuid、simple-uuid、random-32、random-64、random-128、tik） + token-style: uuid + # 是否输出操作日志 + is-log: false + +log: + path: log/ + encoder: UTF-8 diff --git a/src/main/resources/banner.txt b/src/main/resources/banner.txt new file mode 100644 index 0000000..424bd2e --- /dev/null +++ b/src/main/resources/banner.txt @@ -0,0 +1,10 @@ + █ █ █ + █ █ █ + █ █ █ + █▒██▒ ▒███▒ ▒███▒ █▓██ ░███░ ▓██▒ █ ▒█ ███ █▒██▒ ██▓█ + ██ █ █▒ ░█ █▒ ░█ █▓ ▓█ █▒ ▒█ ▓█ ▓ █ ▒█ ▓▓ ▒█ █▓ ▒█ █▓ ▓█ + █ █▒░ █▒░ █ █ █ █░ █▒█ █ █ █ █ █ █ + █ ░███▒ ░███▒ ███ █ █ ▒████ █ ██▓ █████ █ █ █ █ + █ ▒█ ▒█ █ █ █▒ █ █░ █░█░ █ █ █ █ █ + █ █░ ▒█ █░ ▒█ █▓ ▓█ █░ ▓█ ▓█ ▓ █ ░█ ▓▓ █ █ █ █▓ ▓█ + █ ▒███▒ ▒███▒ █▓██ ▒██▒█ ▓██▒ █ ▒█ ███▒ █ █ ██▓█ diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..bda8eac --- /dev/null +++ b/src/main/resources/logback-spring.xml @@ -0,0 +1,240 @@ + + + + + + + + + ${LOG_CONTEXT_NAME} + + + + + + + + + + + + + + + + + + + + + + + + + + + INFO + + + + + + ${CONSOLE_LOG_PATTERN} + + + + UTF-8 + + + + + + + + + + + true + + + + + + ${LOG_HOME}/%d{yyyy-MM-dd}/info/info.%d{yyyy-MM-dd-HH}.%i.log + + + + 30 + + + + + + 10MB + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56}.%method:%L - %msg%n + + utf-8 + + + + + + + + INFO + + ACCEPT + + DENY + + + + + + + + + + + true + + + + + + ${LOG_HOME}/%d{yyyy-MM-dd}/error/error.%d{yyyy-MM-dd}.%i.log + + + + 30 + + + + + + 10MB + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56}.%method:%L - %msg%n + + utf-8 + + + + + + + + ERROR + + ACCEPT + + DENY + + + + + + + + + + + true + + + + ${LOG_HOME}/%d{yyyy-MM-dd}/warn/warn.%d{yyyy-MM-dd}.%i.log + + + + 30 + + + + + + 10MB + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56}.%method:%L - %msg%n + + utf-8 + + + + + + + + WARN + + ACCEPT + + DENY + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/java/HtmlTest.java b/src/test/java/HtmlTest.java new file mode 100644 index 0000000..755ef7b --- /dev/null +++ b/src/test/java/HtmlTest.java @@ -0,0 +1,51 @@ +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Html测试 + * + * @author yanqs + * @since 2024-08-10 + */ +public class HtmlTest { + public static void main(String[] args) throws IOException{ + String str="为什么温泉♨️水那么黄？

















"; + List strings = extractImageUrls(str); + strings.forEach(obj->{ + System.out.println(obj); + }); + + } + + + public static String readHtmlFile(String filePath) throws IOException { + StringBuilder content = new StringBuilder(); + BufferedReader reader = new BufferedReader(new FileReader(filePath)); + String line; + while ((line = reader.readLine()) != null) { + content.append(line); + } + reader.close(); + return content.toString(); + } + + public static List extractImageUrls(String htmlContent) { + List imageUrls = new ArrayList<>(); + String regex = "]*?src\\s*=\\s*['\"]([^'\"]*?)['\"][^>]*?>"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(htmlContent); + while (matcher.find()) { + String imageUrl = matcher.group(1); + imageUrls.add(imageUrl); + } + return imageUrls; + } +} + + + diff --git a/src/test/java/MailTest.java b/src/test/java/MailTest.java new file mode 100644 index 0000000..e3cb3a1 --- /dev/null +++ b/src/test/java/MailTest.java @@ -0,0 +1,14 @@ +import cn.hutool.core.lang.Console; +import cn.hutool.extra.mail.MailAccount; +import com.bcrjl.rss.common.util.MailUtils; + +/** + * @author yanqs + * @since 2024-08-09 + */ +public class MailTest { + public static void main(String[] args) { + MailAccount mailAccount = MailUtils.initMailAccount(); + Console.log(mailAccount); + } +} diff --git a/src/test/java/RSSReader.java b/src/test/java/RSSReader.java new file mode 100644 index 0000000..780f653 --- /dev/null +++ b/src/test/java/RSSReader.java @@ -0,0 +1,40 @@ +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.net.URL; + +/** + * RSS获取 测试类 + * + * @author yanqs + */ +public class RSSReader { + public static void main(String[] args) throws Exception { + // RSS feed URL + // URL rssUrl = new URL("https://rsshub.ys.bcrjl.com/weibo/user/6489032761"); + URL rssUrl = new URL("https://blog.yanqingshan.com/feed/"); + + // Create a DocumentBuilder + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + + // Parse the RSS file + Document document = builder.parse(rssUrl.openStream()); + + // Get all items + NodeList items = document.getElementsByTagName("item"); + + for (int i = 0; i < items.getLength(); i++) { + Element item = (Element) items.item(i); + Element title = (Element) item.getElementsByTagName("title").item(0); + Element link = (Element) item.getElementsByTagName("link").item(0); + + // Print the title + System.out.println(title.getTextContent()); + System.out.println(link.getTextContent()); + } + } +}