node实现简单的数据爬虫-编程知识

node实现简单的数据爬虫

前言

我使用的是墨迹天气的页面，因为这个使用的链接简单页面结构简单并且大都是文字形式

第一步

打开墨迹天气网址随便点开一个页面
在这里插入图片描述
点击F12或者鼠标右键点击检查查看页面的信息

分析页面内容

在这里插入图片描述

使用文字所在的class和标签来定位

在这里插入图片描述

编写代码

配置express环境

引入包

const axios = require(‘axios’);
const cheerio = require(‘cheerio’);

获取html信息

定义url

const weatherURL = https://tianqi.moji.com/weather/china/shanghai/shanghai;

访问相应的网页

try {
let weather = await getWeatherTips(weatherURL);
// let str = $${weather};
// console.log(weather)
res.json(weather);
} catch (error) {
res.status(500).json({ error: ‘Internal Server Error’ });
}
});

将url传入相应的方法中方法进行处理

function getWeatherTips(url) {return new Promise(async (resolve, reject) => {//使用异步请求try {const response = await axios.get(url);//获取url并访问 得到页面const html = response.data || "";//获取页面内容const $ = cheerio.load(html);//解析成htmlconst temp2 = $('.wea_alert  em').text().trim();/const temp = $('.wea_weather em').text().trim() + '℃'; //这就是刚刚看到的class和标签名 获取里面的textconst desc = $('.wea_weather b').text().trim();const water = $('.wea_about span').text().trim();const win = $('.wea_about em').text().trim();const tips = $('.wea_tips em').text().trim();const words = `今日天气\n${desc}\n温度：${temp}\n湿度：${water}\n风力：${win}\n${tips}\n空气质量${temp2}`;// resolve.json({//   success: true// });const word = {temp: temp,desc: desc,water: water,win: win,tips: tips,area: temp2}// resolve.json({//   words: words// });resolve(word);//输出返回的内容// res.json({ success: true, data: 123 });// resolve.json({ success: true, data: `${words}` });// res.json({//   success: true, data: {//     words:words// temp: temp,// desc: desc,// water: water,// win: win,// tips: tips,// }// });} catch (error) {reject(error);}});
}

全部代码

这个是我加了传入参数的版本你可以自己把你定义的天气相应的url设置好

const axios = require('axios');
const cheerio = require('cheerio');app.post('/words', async function (req, res) {const arr = req.body.address;console.log(arr)const weatherURL = `https://tianqi.moji.com/weather/china` + `${arr}`;console.log(weatherURL)try {let weather = await getWeatherTips(weatherURL);// let str = `$${weather}`;// console.log(weather)res.json(weather);} catch (error) {res.status(500).json({ error: 'Internal Server Error' });}
});// 获取墨迹天气提示信息
function getWeatherTips(url) {return new Promise(async (resolve, reject) => {try {const response = await axios.get(url);const html = response.data || "";const $ = cheerio.load(html);const temp2 = $('.wea_alert  em').text().trim();const temp = $('.wea_weather em').text().trim() + '℃';const desc = $('.wea_weather b').text().trim();const water = $('.wea_about span').text().trim();const win = $('.wea_about em').text().trim();const tips = $('.wea_tips em').text().trim();const words = `今日天气\n${desc}\n温度：${temp}\n湿度：${water}\n风力：${win}\n${tips}\n空气质量${temp2}`;// resolve.json({//   success: true// });const word = {temp: temp,desc: desc,water: water,win: win,tips: tips,area: temp2}// resolve.json({//   words: words// });resolve(word);// res.json({ success: true, data: 123 });// resolve.json({ success: true, data: `${words}` });// res.json({//   success: true, data: {//     words:words// temp: temp,// desc: desc,// water: water,// win: win,// tips: tips,// }// });} catch (error) {reject(error);}});
}