【java爬虫】获取个股详细数据并用echarts展示-编程知识

前言

前面一篇文章介绍了获取个股数据的方法，本文将会对获取的接口进行一些优化，并且添加查询数据的接口，并且基于后端返回数据编写一个前端页面对数据进行展示。

具体的获取个股数据的接口可以看上一篇文章

【java爬虫】基于springboot+jdbcTemplate+sqlite+OkHttp获取个股的详细数据-CSDN博客

下面是操作演示，首先是爬虫获取股票数据

接着是进行获取个股详细数据并且进行数据展示

数据图表还可以下载下来，下面是下载下来的图片，不过下载下来的图片就不能查看每个点的详细数据了

后端接口

相对于前文，后端接口进行了一定优化，每年的数据分3次获取，时间段分别是0101-0501,0501-0901和0901-1231，并且每次请求都是从2023年开始逐年往前获取，一旦发现没有数据了就停止获取。

服务类的详细代码如下

@Slf4j
@Service
public class StockService {// 没有数据对应的返回private final String NO_DATA_RESPONSE1 = "historySearchHandler({})";private final String NO_DATA_RESPONSE2 = "history({})";@Autowiredprivate SQLiteStockDao sqLiteStockDao;// 获取一个OKHttp实例private OkHttpClient client = new OkHttpClient().newBuilder().connectTimeout(1000, TimeUnit.SECONDS).build();public void clearAll() {sqLiteStockDao.clearAll();}public void createTbaleIfNotExist() {sqLiteStockDao.createTbaleIfNotExist();}// 查询所有的数据public List<StockEntity> queryAllByCode(String code) {return sqLiteStockDao.queryAllByCode(code);}// 获取数据并且存入数据库// 三个参数分别是：股票代码，开始时间和结束时间// 开始时间和结束时间都填年份，代码中会自动补全具体时间public int getDataByYear(String code, String start, String end) {String url = "https://q.stock.sohu.com/hisHq?";Request request = null;Response response = null;int num = 0;// 一年的数据分三次请求String[] startTime = {"0101", "0501", "0901"};String[] endTime = {"0501", "0901", "1231"};try {for (int i = Integer.parseInt(end); i >= Integer.parseInt(start); i--) {for (int j = startTime.length-1; j >=0; j--) {HttpUrl.Builder httpBuiler = HttpUrl.parse(url).newBuilder();String starttime = i + startTime[j];String endtime = i + endTime[j];log.info("开始计算时间段[" + starttime + "," + endtime + "]内数据");httpBuiler.addQueryParameter("code", "cn_" + code);httpBuiler.addQueryParameter("start", starttime);httpBuiler.addQueryParameter("end", endtime);httpBuiler.addQueryParameter("stat", "1");httpBuiler.addQueryParameter("order", "D");httpBuiler.addQueryParameter("period", "d");httpBuiler.addQueryParameter("callback", "history");httpBuiler.addQueryParameter("rt", "jsonp");request = new Request.Builder().url(httpBuiler.build()).get()   //默认就是GET请求，可以不写.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36").build();response = client.newCall(request).execute();String res = response.body().string();log.info("请求得到的数据：" + res);if (res.contains(NO_DATA_RESPONSE1) || res.contains(NO_DATA_RESPONSE2)) {// 如果返回为空就认为后面没有数据了log.info("时间段[" + starttime + "," + endtime + "]没有数据");return num;} else {List<StockEntity> entities = parseStrToArr(res, code);sqLiteStockDao.insertItems(entities);log.info("时间段[" + starttime + "," + endtime + "]内有" + entities.size() + "条数据");num += entities.size();}}}} catch (IOException e) {e.printStackTrace();}return num;}// 将string数据解析成List列表private List<StockEntity> parseStrToArr(String res, String code) {if (res.contains(NO_DATA_RESPONSE1) || res.contains(NO_DATA_RESPONSE2)) {return new ArrayList<>();}List<StockEntity> entities = new ArrayList<>();res = res.split("\\(\\[")[1].split("]\\)")[0];JSONObject jsonObject = JSON.parseObject(res);// 获取 hq 字段的值Object hq = jsonObject.get("hq");// 判断 hq 的值是否为数组if (hq instanceof JSONArray) {// 遍历数组for (Object arr : (JSONArray) hq) {JSONArray jsonArray = (JSONArray) arr;StockEntity entity = new StockEntity();entity.setRecord_date((String) jsonArray.get(0));Double open_price = Double.parseDouble((String) jsonArray.get(1));Double close_price = Double.parseDouble((String) jsonArray.get(2));Double change_amend = Double.parseDouble((String) jsonArray.get(3));Double change_range = Double.parseDouble(((String) jsonArray.get(4)).split("%")[0]);Double max_price = Double.parseDouble((String) jsonArray.get(5));Double min_price = Double.parseDouble((String) jsonArray.get(6));Double volume = Double.parseDouble((String) jsonArray.get(7));Double turnover = Double.parseDouble((String) jsonArray.get(8));Double turnover_rate = Double.parseDouble(((String) jsonArray.get(9)).split("%")[0]);entity.setOpen_price(open_price);entity.setClose_price(close_price);entity.setChange_amend(change_amend);entity.setChange_range(change_range);entity.setMax_price(max_price);entity.setMin_price(min_price);entity.setVolume(volume);entity.setTurnover(turnover);entity.setTurnover_rate(turnover_rate);entity.setCode(code);entity.setId(entity.getCode() + "_" + (String) jsonArray.get(0));entities.add(entity);}}return entities;}}

Dao层新增了查询某一只股票详细数据的方法，详细代码如下

@Slf4j
@Repository
public class SQLiteStockDao implements StockDao {private final String TABLE_NAME = "stock_table";@Autowiredprivate JdbcTemplate jdbcTemplate;@Overridepublic void clearAll() {String sql = "DELETE FROM " + TABLE_NAME;jdbcTemplate.batchUpdate(sql);log.info("成功清空数据表" + TABLE_NAME);}@Overridepublic void createTbaleIfNotExist() {Integer count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name = ?", Integer.class, TABLE_NAME);if (count == 0) {String sql = "CREATE TABLE " + TABLE_NAME + "(" +"id VARCHAR(50) PRIMARY KEY," +"code VARCHAR(20)," +           // 股票代码"record_date VARCHAR(20)," +    // 记录的时间"open_price float," +           // 开盘价"close_price float," +           // 收盘价"change_ament float," +          // 涨跌额"change_range float," +          // 涨跌幅"max_price float," +             // 最高价格"min_price float," +             // 最低价格"volume float," +                // 成交量（手）"turnover float," +              // 成交额（万）"turnover_rate float)";               // 换手率jdbcTemplate.execute(sql);log.info(TABLE_NAME + "建表成功");} else {log.info("建表失败，表格已存在");}}@Overridepublic void insertItems(List<StockEntity> entityList) {String sql = "INSERT OR IGNORE INTO " + TABLE_NAME + " (id, code, record_date," +"open_price, close_price, change_ament," +"change_range, max_price, min_price," +"volume, turnover, turnover_rate) values (?,?,?,?,?,?,?,?,?,?,?,?)";// 将列表转为Object数组List<Object[]> arr = new ArrayList<>();for(int i=0; i<entityList.size(); i++) {arr.add(entityList.get(i).changeToArray());}jdbcTemplate.batchUpdate(sql, arr);}@Overridepublic List<StockEntity> queryAllByCode(String code) {String sql = "SELECT open_price, close_price, record_date, volume FROM " + TABLE_NAME +" WHERE code=? ORDER BY record_date DESC";log.info("执行sql：" + sql);List<StockEntity> stockEntities = jdbcTemplate.query(sql, new Object[]{code}, new BeanPropertyRowMapper<>(StockEntity.class));return stockEntities;}}

Dao层对应的实体类如下

@Data
@NoArgsConstructor
@AllArgsConstructor
public class StockEntity {private String id;private String code;private String record_date;private Double open_price;private Double close_price;private Double change_amend;private Double change_range;private Double max_price;private Double min_price;private Double volume;private Double turnover;private Double turnover_rate;// 将数据转换为Object数组public Object[] changeToArray() {Object[] arr = new Object[]{id,code,record_date,open_price.toString(),close_price.toString(),change_amend.toString(),change_range.toString(),max_price.toString(),min_price.toString(),volume.toString(),turnover.toString(),turnover_rate.toString()};return arr;}}

最后就是提供给前端调用的接口了，主要是获取某一只股票的数据，只需要传入股票代码就能开始获取数据，还有查询的接口，同样是输入股票代码进行查询，控制类的详细代码如下

@Controller
@CrossOrigin
@RequestMapping("/stock")
public class StockController {private final String START_YEAR = "1985";private final String END_YEAR = "2023";@Autowiredprivate StockService stockService;@RequestMapping("/clear")@ResponseBodypublic String clear() {stockService.clearAll();return "success";}@RequestMapping("/createTable")@ResponseBodypublic String getData() {stockService.createTbaleIfNotExist();return "success";}@RequestMapping("/getDataByYear/{code}/{start}/{end}")@ResponseBodypublic String getDataByYear(@PathVariable("code") String code,@PathVariable("start") String start,@PathVariable("end") String end) {Integer num = stockService.getDataByYear(code, start, end);return num.toString();}@RequestMapping("/getData/{code}")@ResponseBodypublic String getData(@PathVariable("code") String code) {Integer num = stockService.getDataByYear(code, START_YEAR, END_YEAR);List<StockEntity> stockEntityList = stockService.queryAllByCode(code);return JSON.toJSONString(stockEntityList);}@RequestMapping("/queryData/{code}")@ResponseBodypublic String queryData(@PathVariable("code") String code) {List<StockEntity> stockEntityList = stockService.queryAllByCode(code);return JSON.toJSONString(stockEntityList);}
}

前端页面

下面来说一下前端页面的编写，前端页面一共分为三个大块，

沪深300成分股数据和操作按钮，通过按钮可以进行数据获取或者数据展示
个股详细数据，这一个表格的内容会在你选定具体的股票后变更
数据展示，选定个股后会动态生成展示的数据

前端主要用了vue+element-plus+axios+echarts进行编写，echarts表格参数参考了官方示例，由于数据量比较大，所以选用了大数据量的图表，参考的地址如下

Examples - Apache ECharts

下面展示页面的详细代码

<template><div><el-row class="container"><div class="left-grid"><el-card class="box-card"><template #header><div class="card-header"><span>沪深300成分股</span></div></template><el-table:data="table_data":show-header="true":max-height="250"stripe><el-table-columntype="index"label="序号"width="65%"></el-table-column><el-table-columnprop="code"label="股票代码"width="85%"></el-table-column><el-table-columnprop="name"label="公司简称"width="85%"></el-table-column><el-table-column prop="industry" label="操作"><template #default="scope"><el-buttontype="primary"size="small"@click="queryData(scope.row)">查询</el-button><el-buttontype="primary"size="small"@click="getData(scope.row)">获取</el-button></template></el-table-column></el-table></el-card><el-card><template #header><div class="card-header"><span>{{ table_title }}</span></div></template><el-tablev-loading="loading":data="stock_data":show-header="true":max-height="220"stripe><el-table-column prop="record_date" label="时间"></el-table-column><el-table-column prop="open_price" label="开盘价"></el-table-column><el-table-columnprop="close_price"label="收盘价"></el-table-column><el-table-columnprop="volume"label="成交量（手）"></el-table-column></el-table></el-card></div><div class="right-grid" ref="myChart"></div></el-row></div>
</template><script>
import axios from "axios";
import { ElMessage } from "element-plus";
import { getCurrentInstance } from "vue";
export default {data() {return {update_status: "未开始",loading: true,table_title: "个股数据",// 沪深300成分股数据table_data: [],// 个股详细数据stock_data: [],echarts: getCurrentInstance().appContext.config.globalProperties.$echarts,};},mounted() {this.init();},methods: {init() {var url = "http://localhost:9001/queryAll";axios.get(url).then((response) => {this.table_data = response.data;console.log(response);this.loading = false;}).catch((error) => {console.log(error);this.loading = false;});},// 绘制折线图create_axis() {//3.初始化实例对象 echarts.init(dom容器)var data_xAxis = [];var data_yAxis = [];for (var i = this.stock_data.length - 1; i >= 0; i--) {data_xAxis.push(this.stock_data[i].record_date);data_yAxis.push(this.stock_data[i].close_price);}console.log(data_xAxis);console.log(data_yAxis);var dom = this.$refs["myChart"]; // 获取dom节点var myChart = this.echarts.init(dom);//4.指定配置项和数据var option = {tooltip: {trigger: "axis",position: function (pt) {return [pt[0], "10%"];},},title: {left: "center",text: this.table_title,},toolbox: {feature: {dataZoom: {yAxisIndex: "none",},restore: {},saveAsImage: {},},},xAxis: {type: "category",boundaryGap: false,data: data_xAxis,},yAxis: {type: "value",boundaryGap: [0, "100%"],},dataZoom: [{type: "inside",start: 0,end: 10,},{start: 0,end: 10,},],series: [{name: this.table_title,type: "line",symbol: "none",sampling: "lttb",itemStyle: {color: "rgb(135,206,235)",},areaStyle: {color: new this.echarts.graphic.LinearGradient(0, 0, 0, 1, [{offset: 0,color: "rgb(135,206,250)",},{offset: 1,color: "rgb(135,206,235)",},]),},data: data_yAxis,},],};//5.将配置项设置给echarts实例对象，使用刚指定的配置项和数据显示图表。myChart.setOption(option);},// 查询数据queryData(row) {var url = "http://localhost:9001/stock/queryData/" + row.code;this.loading = true;this.table_title = row.code + " " + row.name;ElMessage("开始查询 " + this.table_title + " 的数据");axios.get(url).then((response) => {this.stock_data = response.data;console.log(response);this.loading = false;ElMessage({message: "查询 " + this.table_title + " 的数据成功",type: "success",});// 绘制数据this.create_axis();}).catch((error) => {console.log(error);this.loading = false;ElMessage.error("查询 " + this.table_title + " 的数据失败");});},// 获取数据getData(row) {var url = "http://localhost:9001/stock/getData/" + row.code;this.loading = true;this.table_title = row.code + " " + row.name;ElMessage("开始获取 " + this.table_title + " 的数据");axios.get(url).then((response) => {this.stock_data = response.data;console.log(response);this.loading = false;ElMessage({message: "获取 " + this.table_title + " 的数据成功",type: "success",});// 绘制数据this.create_axis();}).catch((error) => {console.log(error);this.loading = false;ElMessage.error("获取 " + this.table_title + " 的数据失败");});},},
};
</script><style scoped>
.card-header {display: flex;justify-content: space-between;align-items: center;
}
.container {display: grid;grid-template-columns: 35% 65%;width: 100%;height: 80vh;
}
.left-grid {background-color: #f0f0f0;border-radius: 2%;padding: 10px;height: 95%;
}
.right-grid {background-color: #f9ecc3;border-radius: 2%;padding: 10px;height: 95%;
}
</style>

前端页面有一个问题，就是数据量非常非常大，页面会很卡，这个问题的其中一个解决办法就是在获取数据的时候颗粒度可以小一点，比如一个星期获取一个数据之类的，因为一张图表也不可能展示出所有的数据，大家可能也只是想看一个总体的走势图，不过本文没有进行相关的优化，因为个人自用的话这点卡顿是可以接受的。