jsoup登录日志平台后调企业微信机器人自动发送错误日志告警

一、需求:错误日志Top10告警发送

二、需求分解

  1. jsoup实现登录,获取到cookie和token等用户鉴权信息
  2. 获取接口相应的key值
  3. 调用日志平台错误日志Top榜接口,查询到结果集
  4. 调用企业微信机器人发送消息接口
  5. 加上定时任务,可以实现定时发送错误日志告警的功能(后续加进定时任务里面去)

jsoup是java的爬虫框架,可以爬取网页数据,这里没有重点使用,只是做了个登录功能。后续可以专门它写一份爬虫的程序。。

  <dependency><groupId>org.jsoup</groupId><artifactId>jsoup</artifactId><version>1.12.1</version></dependency>
package com.smy.cbs.task;import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.smy.cbs.util.RetryUtil;
import com.smy.framework.core.support.SpringTestCase;
import org.apache.commons.collections4.CollectionUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.junit.Test;
import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
import javax.annotation.Resource;
import javax.net.ssl.*;
import java.io.IOException;
import java.math.BigDecimal;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;/*** @author youlu* @ClassName HttpTestt* @Date 2023/11/22 17:14* @Version V1.0**/
public class HttpTestt2 extends SpringTestCase {@Resourceprivate RetryUtil retryUtil;public static final String LOG_CENTER_BASE_URL = "https://logxxxxx.com/api/v1/";public static final String USER_NAME = "aaaaa";public static final String PASSWORD = "bbbbb";public static String X_Csrf_Token = "";public static String COOKIE = "";public static String id = "";public static int maxCount = Integer.MAX_VALUE;//达到阈值则告警public static List<String> filterContent = Lists.newArrayList();//过滤内容public static int SUB_CONTENT_LENGTH = 240;//截取报错内容字符串长度public static int SUB_TITLE_LENGTH = 20;//截取报错类型字符串长度public static int PERIOD_HOUR = 24*7;//24小时内的错误日志public static List<String> SEARCH_SYSTEM = Lists.newArrayList("cbs_core", "adv_core", "rls_core", "uts_core");//查询的系统//public static List<String> SEARCH_SYSTEM = Lists.newArrayList("adv_core");//查询的系统public static String WX_ROBOT_URL = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=05e1f4a8-aaaaaa" /*+ ",https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=87140cb7-bbbbbb"*/;public static final String TEMPLATE = "{\"markdown\":{\"content\":\"%s\"},\"msgtype\":\"markdown\"}";public static final String QUERY_TEMPLATE = "{\n" +"  \"app\": \"system\",\n" +"  \"source\": \"other\",\n" +"  \"query\": \"repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n| where level=\\\"ERROR\\\"\\n| eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n| stats count() as num by err_type\\n| sort 10 by num\\n| join type=inner err_type [\\n  repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n  | where level=\\\"ERROR\\\"\\n  | eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n  | fields + err_type, _raw\\n  | dedup err_type\\n]\\n| rename _raw as 原始日志, num as 统计, err_type as 错误类型\",\n" +"  \"mode\": \"smart\",\n" +"  \"preview\": false,\n" +"  \"collectSize\": -1,\n" +"  \"timeout\": 1000,\n" +"  \"sorts\": []\n" +"}";public static ThreadPoolExecutor threadPool = new ThreadPoolExecutor(10, 10, 0, TimeUnit.MINUTES,new LinkedBlockingQueue<Runnable>(100), new CustomizableThreadFactory("Log_Center-pool-"));@Testpublic void LogCenterData() throws IOException {//1.模拟登录 jsoupjsoupLogin();for (String systemName : SEARCH_SYSTEM) {try {Thread.sleep(1000);//2.获取keyString id = getKey(systemName);Thread.sleep(8000);List<Map<String, Object>> logCenterList = retryUtil.doRetry(4, () -> {//3.获取查询结果List<Map<String, Object>> contentList = getContentList(systemName, id);if (CollectionUtils.isEmpty(contentList)) {Thread.sleep(8000);throw new Exception(systemName + "未查询到数据,需要重试!");}return contentList;}, systemName + "获取日志数据");//4.调微信发送短信//if ("adv_core".equals(systemName) && CollectionUtils.isNotEmpty(logCenterList)) {//    List<List<Map<String, Object>>> partition = Lists.partition(logCenterList, 5);//    partition.stream().forEach(k -> sendWxMessage(systemName, id, k));//} else {sendWxMessage(systemName, id, logCenterList);//}} catch (Exception e) {e.printStackTrace();}}}public static String getKey(String systemName) {Date endDate = new Date();Date startDate = DateUtil.offsetHour(endDate, -PERIOD_HOUR);//请求参数JSONObject paramJson = JSON.parseObject(String.format(QUERY_TEMPLATE, systemName, systemName));paramJson.put("startTime",startDate.getTime());paramJson.put("endTime",endDate.getTime());HttpResponse execute = HttpRequest.post(LOG_CENTER_BASE_URL + "jobs")//设置请求头(可任意加).header("X-Csrf-Token", X_Csrf_Token).header("Cookie", COOKIE).header("Content-Type", "application/json").header("Connection","keep-alive")//请求参数.body(paramJson.toJSONString()).timeout(40000).execute();String body1 = execute.body();String id = JSON.parseObject(body1).getString("id");return id;}public static List<Map<String,Object>> getContentList(String systemName,String id)  {String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";url += "?_=" + System.currentTimeMillis();HttpResponse execute = HttpRequest.get(url)//设置请求头(可任意加).header("X-Csrf-Token", X_Csrf_Token).header("Cookie", COOKIE).header("Content-Type", "application/json").header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36").header("Sec-Ch-Ua-Platform", "Windows").header("Connection","keep-alive").timeout(40000).execute();JSONArray rows = JSON.parseObject(execute.body()).getJSONArray("rows");System.err.println(rows.toString());List<Map<String, Object>> list = Lists.newArrayList();for (Object row : rows) {try {JSONArray jsonArray = JSON.parseArray(row.toString());String title = jsonArray.get(0).toString();Integer count = Integer.valueOf(jsonArray.get(1).toString());String content = jsonArray.get(2).toString();if (count >= maxCount) {continue;}if (filterContent.stream().anyMatch(k -> content.contains(k))) {continue;}int subContentLen = content.length() <= SUB_CONTENT_LENGTH ? content.length() : SUB_CONTENT_LENGTH;String subContent = content.substring(0, subContentLen);int subTitleLen = title.length() <= SUB_TITLE_LENGTH ? title.length() : SUB_TITLE_LENGTH;String subTitle = content.substring(0, subTitleLen);Map<String, Object> map = new HashMap<>();map.put("title", subTitle);map.put("count", count);map.put("content", subContent);list.add(map);} catch (Exception e) {e.printStackTrace();}}if (CollectionUtils.isEmpty(list)) {System.err.println(systemName + ":未获取到数据,请求链接:" + url + "   token:" + X_Csrf_Token + "   cookie:" + COOKIE);}return list;}public static void sendWxMessage(String systemName, String id, List<Map<String, Object>> contentList) {if (CollectionUtils.isEmpty(contentList)) {System.err.println(systemName + ":发送内容为空,不发送机器人微信消息,id:" + id);return;}String periodDesc = getPeriodDesc();final int[] topNum = {1};String StringContent = contentList.stream().map(k -> {String title = (String) k.get("title");Integer count = (Integer) k.get("count");String content = (String) k.get("content");//return String.format("> top-%d:%s\n> 出现次数:%d\n> 错误详情描述: %s", topNum[0]++, title, count, content.trim());return String.format("> top-%d:出现次数:%d\n> 错误详情描述: %s", topNum[0]++, count, content.trim());}).collect(Collectors.joining("\n\n"));String content = String.format("%s近%s内错误日志Top10\n%s", systemName, periodDesc, StringContent);String sendContent = String.format(TEMPLATE, content);////Map<String, String> contentMap = new HashMap<>();//contentMap.put("content", content);//Map<String, String> markdownMap = new HashMap<>();//markdownMap.put("markdown", JSON.toJSONString(contentMap));//markdownMap.put("msgtype", "markdown");//String sendText = JSON.toJSONString(markdownMap);////System.err.println(sendText);Arrays.stream(WX_ROBOT_URL.split(",")).forEach(wx -> {//String post = HttpUtil.post(wx + "&debug=1", sendText);//String post2 = HttpUtil.post(wx + "&debug=1", sendContent);String post2 = HttpUtil.post(wx, sendContent);System.err.println(systemName + "发送微信情况2" + post2);//System.err.println(systemName + "发送微信情况" + post);});}private static String getPeriodDesc() {if (PERIOD_HOUR <= 24) {return PERIOD_HOUR + "小时";}BigDecimal dayBigDecimal = BigDecimal.valueOf(PERIOD_HOUR).divide(BigDecimal.valueOf(24), 2, BigDecimal.ROUND_HALF_UP);String s = StrUtil.removeSuffix(StrUtil.removeSuffix(String.valueOf(dayBigDecimal), "00"), "0");String[] split = s.split("\\.");if (split.length == 1) {return split[0] + "天";}return s + "天";}/*** Jsoup 模拟登录 访问个人中心* 先构造登录请求参数,成功后获取到cookies* 设置request cookies,再次请求* @throws IOException*/public static void jsoupLogin() throws IOException {//Jsoup加这个,避免请求https报证书问题trustEveryone();// 构造登陆参数Map<String,String> data = new HashMap<>();data.put("username", USER_NAME);data.put("password", PASSWORD);Connection.Response response = Jsoup.connect(LOG_CENTER_BASE_URL + "account/ldap/login").ignoreContentType(true) // 忽略类型验证.ignoreHttpErrors(true).followRedirects(false) // 禁止重定向.postDataCharset("utf-8").header("Upgrade-Insecure-Requests","1").header("Accept","application/json").header("Content-Type","application/json").header("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36").requestBody(JSON.toJSONString(data)).method(Connection.Method.POST).execute();response.charset("UTF-8");// login 中已经获取到登录成功之后的cookies// 构造访问个人中心的请求Map<String, String> cookies = response.cookies();cookies.forEach((k, v) -> COOKIE = k + "=" + v);String body = response.body();X_Csrf_Token = JSON.parseObject(body).getString("X-Csrf-Token");System.err.println("COOKIE:" + COOKIE);System.err.println("X_Csrf_Token:" + X_Csrf_Token);}public static void jsoupHandle(String id) throws IOException {String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";Document document = Jsoup.connect(url).header("X-Csrf-Token", X_Csrf_Token).header("Cookie", COOKIE).ignoreContentType(true) // 忽略类型验证.ignoreHttpErrors(true).method(Connection.Method.GET).get();System.err.println(url);String s = JSON.toJSONString(document);System.err.println(s);}public static void trustEveryone() {try {HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {public boolean verify(String hostname, SSLSession session) {return true;}});SSLContext context = SSLContext.getInstance("TLS");context.init(null, new X509TrustManager[] { new X509TrustManager() {public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {}public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {}public X509Certificate[] getAcceptedIssuers() {return new X509Certificate[0];}} }, new SecureRandom());HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());} catch (Exception e) {// e.printStackTrace();}}}

三、实现效果

企业微信机器狗开发者文档:群机器人配置说明 - 接口文档 - 企业微信开发者中心

本文来自互联网用户投稿,该文观点仅代表作者本人,不代表本站立场。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如若转载,请注明出处:http://www.hqwc.cn/news/224967.html

如若内容造成侵权/违法违规/事实不符,请联系编程知识网进行投诉反馈email:809451989@qq.com,一经查实,立即删除!

相关文章

Web3.0时代:区块链DAPP将如何颠覆传统模式

小编介绍&#xff1a;10年专注商业模式设计及软件开发&#xff0c;擅长企业生态商业模式&#xff0c;商业零售会员增长裂变模式策划、商业闭环模式设计及方案落地&#xff1b;扶持10余个电商平台做到营收过千万&#xff0c;数百个平台达到百万会员&#xff0c;欢迎咨询。 随着…

CCC联盟数字车钥匙(四)——UWB MAC协议

本篇文章介绍CCC协议中关于UWB消息流&#xff0c;以及相关会话设置及控制消息的配置。 5、MAC协议 5.1 测距交换序列 本节介绍DK MAC层协议&#xff0c;针对双边双向测距方法&#xff08;DS-TWR&#xff09;&#xff0c;协调器和Responder之间的数据交换方式。 图中关于SP0、…

解决git action发布报错:Input required and not supplied: upload_url

现象&#xff1a; 这个问题死活都找不到原因&#xff0c;后来打了一段调试的代码 - name: Debug Create Release Output run: | echo "Release ID: ${{ env.RELEASE_ID }}" echo "Release Upload URL: ${{ env.RELEASE_UPLOAD_URL }}" env: RELEASE_ID: ${…

统信UOS安装Virtualbox虚拟机和Windows10系统

在UOS统信系统中部署Windows环境我可以通过安装虚拟机来实现&#xff0c;这也可以解决软件不适配带来的一些问题&#xff0c;当然对硬件配置也有一定的要求&#xff0c;不建议性能过低的设备使用。 接下来请按照以下步骤进行安装Virtualbox及Win10虚拟系统的设置。 1、安装Vi…

深度学习【二】

1.运行时错误 1.1 ModuleNotFoundError: No module named ‘torch_scatter’ 参考 https://blog.csdn.net/weixin_42421914/article/details/132875571 pip install --no-index torch-scatter -f https://pytorch-geometric.com/whl/torch-1.13.1%2Bcpu.html

OSG粒子系统与阴影 - ​​​​​​​阴影shadow(7)

OSG阴影 在虚拟现实仿真中&#xff0c;为了真实地模拟自然效果&#xff0c;阴影效果是不可缺少的&#xff0c;它对一个场景的真实性是非常重要的。在游戏或仿真中&#xff0c;一个高效的阴影往往能够提供非常强悍的视觉真实感。 osgShadow库 在OSG中专门定义了一个名字空间osg…

lua的gc原理

lua垃圾回收(Garbage Collect)是lua中一个比较重要的部分。由于lua源码版本变迁&#xff0c;目前大多数有关这个方面的文章都还是基于lua5.1版本&#xff0c;有一定的滞后性。因此本文通过参考当前的5.3.4版本的Lua源码&#xff0c;希望对Lua的GC算法有一个较为详尽的探讨。 L…

【LeetCode】每日一题 2023_11_28 设计前中后队列(数组/链表/双端队列)

文章目录 刷题前唠嗑题目&#xff1a;设计前中后队列题目描述代码与解题思路偷看大佬题解 结语 刷题前唠嗑 LeetCode&#xff1f;启动&#xff01;&#xff01;&#xff01; 这道题的难度&#xff0c;才是我想象中的中等题的难度好吧&#xff0c;昨天那玩意对我来说还是太难了…

9个AI视频后期处理神器——Runway 全功能超详细使用教程(3)

前面2期内容&#xff0c;主要给大家重点介绍了Runway视频生成技术的核心产品功能板块Gen1、Gen2、FI使用教程&#xff0c;还没有看过的小伙伴可以回看往期文章。除了视频生成AI技术外&#xff0c;Runway还具有图片、视频后期处理30多项单个功能&#xff0c;例如视频修复、视频主…

封装可多选的组件(Autocomplete)

一。组件库Material UI 1.1 地址 https://v4.mui.com/zh/getting-started/installation/ 1.2 简介 自称世界上最受欢迎的React UI组件库(能看到这里的基本用法应该都清楚了&#xff0c;我就不重复了) 二。效果展示 三。代码展示 import React from reactimport { useField, us…

阿里云ACE认证之国际版与国内版对比!

大厂疯狂裁员&#xff0c;互联网行业迎来寒冬&#xff0c;技术人员被动陷入疯狂内卷。在愈加内卷的IT领域&#xff0c;“云计算”作为少有的蓝海&#xff0c;无疑是打工人未来实现职场提升、摆脱内卷的绝佳选择&#xff01; 对于云计算行业的人来说&#xff0c;最值得考的肯定是…

rust-flexi_logger

flexi_logger 是字节开源的rust日志库。目前有log4rs、env_log 等库&#xff0c;综合比较下来&#xff0c;还是flexi_logger简单容易上手&#xff0c;而且自定义很方便&#xff0c;以及在效率方面感觉也会高&#xff0c;下篇文章我们来测试下。 下面来看下怎么使用 关注 vx gol…