MySQL数据库导入100万数据不同方式的性能差异-编程知识

本文将介绍MySQL数据库导入100万数据的三种方式性能比较。

三种方式分别为：

（1）逐条INSERT

（2）批量INSERT提交

（3）通过mysql自带的load data命令

应用场景：假设需要向100万个号码发送短信，收到了100万个号码txt文件，现需要将号码导入到数据库，通过短信系统发送短信。向数据库导入数据的场景还有很多，比如在新老系统切换的时候，需要将老系统的数据迁移导入到新系统，这些都可能涉及到大量数据导入的问题。

首先，我们先准备数据库表、100万个号码、生成insert语句的java代码、批量向数据库提交insert语句的java代码。

（1）数据库表如下：

(2) 生成100万个号码

package com.fd.demo.common;import org.springframework.stereotype.Component;import java.io.*;/*** @Author: thinkpad* @Date: 2023-12-30 9:23*/
@Component
public class TelephoneGenerate {/*** 生成号码个数* @param count*/public void generate(int count) throws Exception{String telephoneBase = "13900000000";if(count > 100000000){throw new Exception("号码生成个数最大不能超过1亿");}String fileName = "telephoneGenerate.txt";File file = new File(fileName);if(!file.exists()){file.createNewFile();}// demo代码 不考虑异常情况FileOutputStream fileOutputStream = new FileOutputStream(file);OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream);BufferedWriter bufferedWriter = new BufferedWriter(outputStreamWriter);for(int i = 0; i < count; i++){String str = String.valueOf(i);String telephone = telephoneBase.substring(0, 11-str.length()) + str;bufferedWriter.write(telephone);bufferedWriter.write("\n");}bufferedWriter.flush();bufferedWriter.close();outputStreamWriter.close();fileOutputStream.close();}public static void main(String[] args) throws Exception {TelephoneGenerate telephoneGenerate = new TelephoneGenerate();long beginTime = System.currentTimeMillis();telephoneGenerate .generate(1000000);long endTime = System.currentTimeMillis();System.out.println("cost time: " + (endTime - beginTime) + "ms");}
}

(3) 生成100万条insert语句

package com.fd.demo.common;import org.springframework.stereotype.Component;import java.io.*;
import java.util.Objects;/*** @Author: thinkpad* @Date: 2023-12-30 9:22*/
@Component
public class MySQLInsertGenerate {/*** 生成insert语句*/public void generateInsertSQL() throws Exception{String fileName = "telephoneGenerate.txt";File file = new File(fileName);// demo代码 不考虑异常情况FileInputStream fileInputStream = new FileInputStream(file);InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream);BufferedReader bufferedReader = new BufferedReader(inputStreamReader);String sqlFileName = "telephoneInsertSQL.txt";File sqlFile = new File(sqlFileName);FileOutputStream fileOutputStream = new FileOutputStream(sqlFile);OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream);BufferedWriter bufferedWriter = new BufferedWriter(outputStreamWriter);String telephone = bufferedReader.readLine();while(!Objects.isNull(telephone) && !"".equals(telephone)){TelephoneRecvListVO telephoneRecvListVO = new TelephoneRecvListVO();telephoneRecvListVO.setId(IdWorker.nextId()); //雪花算法生成idtelephoneRecvListVO.setAppId(2);telephoneRecvListVO.setAppName("demo");telephoneRecvListVO.setTemplateId(2);telephoneRecvListVO.setTemplateContent("测试短信内容发送");telephoneRecvListVO.setTelephoneList(telephone); // 如果有多个号码，以短号隔开telephoneRecvListVO.setCreateDate("20231230");telephoneRecvListVO.setCreateTime("170000");telephoneRecvListVO.setStatus("9"); // 9初始状态 0已发送 1发送失败 2发送中String insertSql = generate(telephoneRecvListVO);bufferedWriter.write(insertSql);bufferedWriter.write("\n");telephone = bufferedReader.readLine();}bufferedWriter.flush();bufferedWriter.close();outputStreamWriter.close();fileOutputStream.close();inputStreamReader.close();fileInputStream.close();}// 可以使用反射进行组装private String generate(TelephoneRecvListVO telephoneRecvListVO){StringBuilder sb = new StringBuilder();sb.append("insert into recv_list(ID, APP_ID, APP_NAME, TEMPLATE_ID, TEMPLATE_CONTENT, TELEPHONE_LIST, CREATE_DATE, CREATE_TIME, STATUS) values(");sb.append(telephoneRecvListVO.getId());sb.append(",");sb.append(telephoneRecvListVO.getAppId());sb.append(",");sb.append("'" + telephoneRecvListVO.getAppName() + "'");sb.append(",");sb.append(telephoneRecvListVO.getTemplateId());sb.append(",");sb.append("'" + telephoneRecvListVO.getTemplateContent() + "'" );sb.append(",");sb.append("'" + telephoneRecvListVO.getTelephoneList() + "'" );sb.append(",");sb.append("'" + telephoneRecvListVO.getCreateDate() + "'");sb.append(",");sb.append("'" + telephoneRecvListVO.getCreateTime() + "'");sb.append(",");sb.append("'" + telephoneRecvListVO.getStatus() + "'");sb.append(");");return sb.toString();}public static void main(String[] args) throws Exception {MySQLInsertGenerate generate = new MySQLInsertGenerate();long beginTime = System.currentTimeMillis();generate.generateInsertSQL();long endTime = System.currentTimeMillis();System.out.println("cost time: " + (endTime - beginTime) + " ms");}
}

（4）单笔向recv_list插入数据

package com.fd.demo.mysqlinsert;import java.io.*;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.Objects;/*** @Author: thinkpad* @Date: 2023-12-30 19:07*/
public class SingleInsert {public void singleInsert() throws Exception {File file = new File("telephoneInsertSQL.txt");FileInputStream fileInputStream = new FileInputStream(file);InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream);BufferedReader bufferedReader = new BufferedReader(inputStreamReader);try{Connection conn = MySqlJDBC.getConnection();String sql = bufferedReader.readLine();long currentIndex = 1;while(!Objects.isNull(sql) && !"".equals(sql)){if(currentIndex%10000 == 0){System.out.println("Current Index = " + currentIndex);}PreparedStatement ps = conn.prepareStatement(sql);ps.execute();sql = bufferedReader.readLine();currentIndex++;}}catch (Exception e){e.printStackTrace();}bufferedReader.close();inputStreamReader.close();fileInputStream.close();}public static void main(String[] args) throws Exception {SingleInsert singleInsert = new SingleInsert();long beginTime = System.currentTimeMillis();singleInsert.singleInsert();long endTime = System.currentTimeMillis();System.out.println("cost time: " + (endTime - beginTime) + " ms");}
}

单笔总共花费时间：将近31小时

（5）批量向recv_list插入数据（每10000笔提交一次事务）

package com.fd.demo.mysqlinsert;import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.Statement;
import java.util.Objects;/*** @Author: thinkpad* @Date: 2023-12-30 19:23*/
public class BatchInsert {public void batchInsert() throws Exception{File file = new File("telephoneInsertSQL.txt");FileInputStream fileInputStream = new FileInputStream(file);InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream);BufferedReader bufferedReader = new BufferedReader(inputStreamReader);try{Connection conn = MySqlJDBC.getConnection();Statement statement = conn.createStatement();conn.setAutoCommit(false); // 批量插入主要是不自动提交事务，String sql = bufferedReader.readLine();long currentIndex = 1;while(!Objects.isNull(sql) && !"".equals(sql)){statement.executeUpdate(sql);if(currentIndex % 10000 == 0){System.out.println("currentIndex = " + currentIndex);conn.commit();}sql = bufferedReader.readLine();currentIndex++;}}catch (Exception e){e.printStackTrace();}bufferedReader.close();inputStreamReader.close();fileInputStream.close();}public static void main(String[] args) throws Exception {BatchInsert batchInsert = new BatchInsert();long beginTime = System.currentTimeMillis();batchInsert.batchInsert();long endTime = System.currentTimeMillis();System.out.println("cost time: " + (endTime - beginTime) + " ms");}
}

批量插入数据，仅花费了4分钟，完全秒杀单笔插入的效率。

（6）使用mysql的load data命令

load data infile 'G:/idea_workspace/MySQLPerformance/telephoneImportSQL.txt' 
into table recv_list fields terminated by ',' enclosed by '"' 
lines terminated by '\n';

如果在执行的过程中发生以下错误：

ERROR 1261 (01000):Row 1 doesn't contain data for all colums

这是因为sql_mode 被设为了 strict 模式, 要想继续导入需要把“strict_trans_tables” 从 sql_mode 中去掉.

查看 MySQL 当前连接的 sql_mode