C#读取一个百万条数据的文件，同时批量一次性导入sqlitedb，需要花费多长时间

在这里插入图片描述
读取的代码：

      public void CSV2DataTableTest(string fileName){FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read);StreamReader sr = new StreamReader(fs, new UnicodeEncoding());//记录每次读取的一行记录string strLine = "";//记录每行记录中的各字段内容string[] aryLine;//标示列数int columnCount = 0;//标示是否是读取的第一行bool IsFirst = true;List<string> Columns = new List<string>();Columns.Add("Id INTEGER PRIMARY KEY AUTOINCREMENT  NOT NULL");List<List<string>> dtList = new List<List<string>>();while ((strLine = sr.ReadLine()) != null){aryLine = strLine.Split(',');if (IsFirst == true){IsFirst = false;columnCount = aryLine.Length;//创建列for (int i = 0; i < columnCount; i++){Columns.Add((string)aryLine[i]);}}else{dtList.Add(aryLine.ToList());}}sr.Close();fs.Close();sr.Dispose();fs.Dispose();var insertColumns = new List<string>();insertColumns.AddRange(Columns);insertColumns[0] = "Id";SqliteHelper helper = new SqliteHelper();string table = "AA3";if (!helper.TableExist(table))helper.CreateTable(table, Columns);else{helper.ExecuteNonQuery("delete from AA3");helper.ExecuteNonQuery("update sqlite_sequence SET seq = 0 where name ='AA3';");}var dateStart = DateTime.Now;helper.OpenConnection();helper.InsertDataTest(dtList, insertColumns, table);helper.CloseConnection();string times = "导入sqlite花费时间" + (DateTime.Now - dateStart).TotalMilliseconds + "ms";label2.Text = times;dtList.Clear();GC.Collect();}

插入sqlite代码：


using NPOI.SS.Formula.Functions;
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.Common;
using System.Data.SQLite;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;namespace TestExpression
{public class SqliteHelper{// string db = "Data Source=:memory:";//连接字符串//   private static string str = "Data source=" + AppDomain.CurrentDomain.BaseDirectory + "\\data.db";private static string str = "Data source=" + AppDomain.CurrentDomain.BaseDirectory + "\\D518DCC2-149A-48CC-8355-682B0BF92D26.DB";/// <summary>/// 增删改/// 20180723/// </summary>/// <param name="sql">sql语句</param>/// <param name="param">sql参数</param>/// <returns>受影响的行数</returns>public static int ExecuteNonQuery(string sql, params SQLiteParameter[] param){try{using (SQLiteConnection con = new SQLiteConnection(str)){using (SQLiteCommand cmd = new SQLiteCommand(sql, con)){con.Open();if (param != null){cmd.Parameters.AddRange(param);}string sql2 = cmd.CommandText;//  con.Close();return cmd.ExecuteNonQuery();}}}catch (SQLiteException se){return 0;}}/// <summary>/// 创建表/// </summary>/// <param name="dbPath">指定数据库文件</param>/// <param name="tableName">表名称</param>public void CreateTable(string table, List<string> Columns){using (SQLiteConnection con = new SQLiteConnection(str)){con.Open();string Column = "";for (int i = 0; i < Columns.Count; i++){Column += Columns[i] + ",";}Column = Column.Substring(0, Column.Length - 1);SQLiteCommand cmd = new SQLiteCommand();cmd.Connection = con;cmd.CommandText = " CREATE TABLE " + table + "(" + Column + ")";cmd.ExecuteNonQuery();con.Close();}}/// <summary>/// 判断表是否存在/// </summary>/// <param name="tableName"></param>/// <returns></returns>public bool TableExist(string table){using (SQLiteConnection con = new SQLiteConnection(str)){con.Open();SQLiteCommand mDbCmd = con.CreateCommand();mDbCmd.CommandText = "SELECT COUNT(*) FROM sqlite_master where type='table' and name='" + table + "';";int row = Convert.ToInt32(mDbCmd.ExecuteScalar());con.Close();if (0 < row){return true;}else{return false;}}}/// <summary>/// 增删改/// 20180723/// </summary>/// <param name="sql">sql语句</param>/// <param name="param">sql参数</param>/// <returns>受影响的行数</returns>public int ExecuteNonQuery(string sql){try{using (SQLiteConnection con = new SQLiteConnection(str)){using (SQLiteCommand cmd = new SQLiteCommand(sql, con)){con.Open();string sql2 = cmd.CommandText;//con.Close();return cmd.ExecuteNonQuery();}}}catch (SQLiteException se){return 0;}}SQLiteConnection SqlConnection = null;public void OpenConnection(){SqlConnection = new SQLiteConnection(str);SqlConnection.Open();}public void CloseConnection(){SqlConnection.Close();}public void InsertDataTest(List<List<string>> dt, List<string> Columns, string tableName){SQLiteBulkInsert sbi = new SQLiteBulkInsert(SqlConnection, tableName);sbi.AddParameter("Id", DbType.Int32);for (int i = 1; i < Columns.Count; i++){sbi.AddParameter(Columns[i], DbType.String);}for (int j = 0; j < dt.Count; j++){object[] objects = new object[Columns.Count];objects[0] = null;for (int i = 1; i < Columns.Count; i++){objects[i]= dt[j][i-1];}sbi.Insert(objects);}sbi.Flush();}public int InsertDataBulk3(List<List<string>> dt, List<string> Columns, string tableName){using (SQLiteTransaction tran = SqlConnection.BeginTransaction()){try{string columnStr = string.Join(",", Columns);string columnStr2 = "@" + string.Join(",@", Columns);using (SQLiteCommand command = new SQLiteCommand("Insert into " + tableName + "(" + columnStr + ") values(" + columnStr2 + ")", SqlConnection)){for (int j = 0; j < dt.Count; j++){var drData = dt[j];command.Parameters.Add(new SQLiteParameter("@Id", null));for (int i = 1; i < Columns.Count; i++){command.Parameters.Add(new SQLiteParameter("@" + Columns[i], drData[i - 1]));}command.ExecuteNonQuery();command.Parameters.Clear();}}tran.Commit();return 0;}catch (Exception ex){tran.Rollback();return 1;}}}public int InsertDataBulk(DataTable dt, List<string> Columns, string tableName){using (SQLiteTransaction tran = SqlConnection.BeginTransaction()){try{string columnStr = string.Join(",", Columns);string columnStr2 = "@" + string.Join(",@", Columns);using (SQLiteCommand command = new SQLiteCommand("Insert into " + tableName + "(" + columnStr + ") values(" + columnStr2 + ")", SqlConnection)){foreach (DataRow drData in dt.Rows){command.Parameters.Add(new SQLiteParameter("@Id", null));for (int i = 1; i < Columns.Count; i++){command.Parameters.Add(new SQLiteParameter("@" + Columns[i], drData[Columns[i]]));}command.ExecuteNonQuery();command.Parameters.Clear();}}tran.Commit();return 0;}catch (Exception ex){tran.Rollback();return 1;}}}public int UpdateDataBulk(Dictionary<int, string?> dt, string changeColumn, string tableName){using (SQLiteTransaction tran = SqlConnection.BeginTransaction()){try{using (SQLiteCommand command = new SQLiteCommand("update " + tableName + " set " + changeColumn + "=@" + changeColumn + " where Id=@Id", SqlConnection)){foreach (var drData in dt){command.Parameters.Add(new SQLiteParameter("@" + changeColumn, drData.Value));command.Parameters.Add(new SQLiteParameter("@Id", drData.Key));command.ExecuteNonQuery();command.Parameters.Clear();}}tran.Commit();return 0;}catch (Exception ex){tran.Rollback();return 1;}}}/// <summary>/// 查询/// 20180723/// </summary>/// <param name="sql">sql语句</param>/// <param name="param">sql参数</param>/// <returns>首行首列</returns>public object ExecuteScalarFirst(string sql){try{using (SQLiteConnection con = new SQLiteConnection(str)){using (SQLiteCommand cmd = new SQLiteCommand(con)){con.Open();cmd.CommandText = sql;return cmd.ExecuteScalar();}}}catch (Exception ex){return null;}}/// <summary>/// 查询/// </summary>/// <param name="sql">sql语句</param>/// <param name="param">sql参数</param>/// <returns>首行首列</returns>public object ExecuteScalar(string sql, params SQLiteParameter[] param){using (SQLiteConnection con = new SQLiteConnection(str)){using (SQLiteCommand cmd = new SQLiteCommand(sql, con)){con.Open();if (param != null){cmd.Parameters.AddRange(param);}return cmd.ExecuteScalar();}}}/// <summary>/// 多行查询/// </summary>/// <param name="sql">sql语句</param>/// <param name="param">sql参数</param>/// <returns>SQLiteDateReader</returns>public SQLiteDataReader ExecuteReader(string sql, params SQLiteParameter[] param){using (SQLiteConnection con = new SQLiteConnection(str)){using (SQLiteCommand cmd = new SQLiteCommand(sql, con)){if (param != null){cmd.Parameters.AddRange(param);}try{con.Open();return cmd.ExecuteReader(System.Data.CommandBehavior.CloseConnection);}catch (Exception ex){con.Close();con.Dispose();throw ex;}}}}/// <summary>/// 查询多行数据/// </summary>/// <param name="sql">sql语句</param>/// <param name="param">sql参数</param>/// <returns>一个表</returns>public DataTable ExecuteTable(string sql, params SQLiteParameter[] param){DataTable dt = new DataTable();using (SQLiteDataAdapter sda = new SQLiteDataAdapter(sql, str)){if (param != null){sda.SelectCommand.Parameters.AddRange(param);}sda.Fill(dt);}return dt;}/// <summary>/// 数据插入/// 20180725/// </summary>/// <param name="tbName">表名</param>/// <param name="insertData">需要插入的数据字典</param>/// <returns>受影响行数</returns>public int ExecuteInsert(string tbName, Dictionary<String, String> insertData){string point = "";//分隔符号(,)string keyStr = "";//字段名拼接字符串string valueStr = "";//值的拼接字符串List<SQLiteParameter> param = new List<SQLiteParameter>();foreach (string key in insertData.Keys){keyStr += string.Format("{0} `{1}`", point, key);valueStr += string.Format("{0} @{1}", point, key);param.Add(new SQLiteParameter("@" + key, insertData[key]));point = ",";}string sql = string.Format("INSERT INTO `{0}`({1}) VALUES({2})", tbName, keyStr, valueStr);//return sql;return ExecuteNonQuery(sql, param.ToArray());}/// <summary>/// 执行Update语句/// 20180725/// </summary>/// <param name="tbName">表名</param>/// <param name="where">更新条件：id=1</param>/// <param name="insertData">需要更新的数据</param>/// <returns>受影响行数</returns>public int ExecuteUpdate(string tbName, string where, Dictionary<String, String> insertData){string point = "";//分隔符号(,)string kvStr = "";//键值对拼接字符串(Id=@Id)List<SQLiteParameter> param = new List<SQLiteParameter>();foreach (string key in insertData.Keys){kvStr += string.Format("{0} {1}=@{2}", point, key, key);param.Add(new SQLiteParameter("@" + key, insertData[key]));point = ",";}string sql = string.Format("UPDATE `{0}` SET {1} WHERE {2}", tbName, kvStr, where);return ExecuteNonQuery(sql, param.ToArray());}/// <summary>/// 查询/// 20180723/// </summary>/// <param name="sql">sql语句</param>/// <param name="param">sql参数</param>/// <returns>首行首列</returns>public DataTable GetDataTable(string sql){try{using (SQLiteConnection con = new SQLiteConnection(str)){using (SQLiteCommand cmd = new SQLiteCommand(sql, con)){con.Open();SQLiteDataAdapter ad = new SQLiteDataAdapter(cmd);DataTable tb = new DataTable();ad.Fill(tb);con.Close();return tb;}}}catch (Exception ex){MessageBox.Show(ex.Message);return null;}}}
}

本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如若转载，请注明出处：http://www.hqwc.cn/news/435031.html

如若内容造成侵权/违法违规/事实不符，请联系编程知识网进行投诉反馈email:809451989@qq.com，一经查实，立即删除！

相关文章

大创项目推荐 题目：基于卷积神经网络的手写字符识别 - 深度学习

大创项目推荐 题目：基于卷积神经网络的手写字符识别 - 深度学习

文章目录 0 前言1 简介2 LeNet-5 模型的介绍2.1 结构解析2.2 C1层2.3 S2层S2层和C3层连接 2.4 F6与C5层 3 写数字识别算法模型的构建3.1 输入层设计3.2 激活函数的选取3.3 卷积层设计3.4 降采样层3.5 输出层设计 4 网络模型的总体结构5 部分实现代码6 在线手写识别7 最后 0 前言…
阅读更多...
Java基于 SpringBoot+Vue 的高校心理教育辅导系统的研究与实现

Java基于 SpringBoot+Vue 的高校心理教育辅导系统的研究与实现

博主介绍&#xff1a;✌程序员徐师兄、7年大厂程序员经历。全网粉丝30W、csdn博客专家、掘金/华为云/阿里云/InfoQ等平台优质作者、专注于Java技术领域和毕业项目实战✌ &#x1f345;文末获取源码联系&#x1f345; &#x1f447;&#x1f3fb; 精彩专栏推荐订阅&#x1f447;…
阅读更多...
后端学习：数据库MySQL学习

后端学习：数据库MySQL学习

数据库简介 数据库&#xff1a;英文为 DataBase&#xff0c;简称DB&#xff0c;它是存储和管理数据的仓库。   接下来&#xff0c;我们来学习Mysql的数据模型&#xff0c;数据库是如何来存储和管理数据的。在介绍 Mysql的数据模型之前&#xff0c;需要先了解一个概念&#xf…
阅读更多...
线性代数----------学习记录

线性代数----------学习记录

线性代数发展历程 &#xff08;1&#xff09;线性方程组&#xff1a;例如二元一次方程组&#xff1b; &#xff08;2&#xff09;行列式&#xff1a;determinant,克莱默&#xff0c;莱布尼兹&#xff1b; &#xff08;3&#xff09;矩阵&#xff1a;方程个数与未知数的个数可…
阅读更多...
CMake简明教程 笔记

CMake简明教程 笔记

推荐B站视频&#xff1a;1.1 Cmake构建项目的流程_哔哩哔哩_bilibilihttps://www.bilibili.com/video/BV1xa4y1R7vT?p1&vd_sourcea934d7fc6f47698a29dac90a922ba5a3 >>目录 1&#xff09;CMake初体验 CMake构建流程Windows下使用CMake构建项目Linux下使用CMake构…
阅读更多...
Django实战

Django实战

一、开发登录表单 def login_form(request):html = <html><body><form method="post">用户名:<input name = "username" type="text"></input></br>密码：<input name = "password" type = &q…
阅读更多...
蓝牙----蓝牙GAP层

蓝牙----蓝牙GAP层

蓝牙协议栈----GAP GAP的角色连接过程连接参数 GAP&#xff1a;通用访问配置协议层 gap的角色发现的模式与过程连接模式与过程安全模式与过程 CC2640R2F的GAP层抽象 GAP的角色 Broadcaster 广播电台 -不可连接的广播者。Observer 观察者 -扫描广播者但无法启动连接。Periphe…
阅读更多...
【T6】畅捷通T6软件后， 打开总账模块，提示：3343不可识别的数据库格式。

【T6】畅捷通T6软件后， 打开总账模块，提示：3343不可识别的数据库格式。

【问题描述】 登录畅捷通T6软件后&#xff0c; 打开总账模块&#xff0c;提示&#xff1a;3343不可识别的数据库格式 C: Windows\system32\UFCOMSQL\Ufformat.mdb。 【解决方法】 找到相同系统、软件版本正常的环境下&#xff0c; 将【Ufformat.mdb】文件进行覆盖。 【附件文…
阅读更多...
python222网站实战(SpringBoot+SpringSecurity+MybatisPlus+thymeleaf+layui)-友情链接管理实现

python222网站实战(SpringBoot+SpringSecurity+MybatisPlus+thymeleaf+layui)-友情链接管理实现

锋哥原创的SpringbootLayui python222网站实战&#xff1a; python222网站实战课程视频教程&#xff08;SpringBootPython爬虫实战&#xff09; ( 火爆连载更新中... )_哔哩哔哩_bilibilipython222网站实战课程视频教程&#xff08;SpringBootPython爬虫实战&#xff09; ( 火…
阅读更多...
el-upload上传文件，如何使用action、auto-upload

el-upload上传文件，如何使用action、auto-upload

1点击打开自动上传 没有 :auto-upload"false"就是自动上传 2点击上传按钮上传 不自动上传:auto-upload"false" <el-dialogstyle"background: rgb(18, 67, 112); border: 1px solid #409eff":title"上传文件":model-value"…
阅读更多...
JavaEE-自定义SSM-编写核心- my spring bean工厂（IoC、DI）

JavaEE-自定义SSM-编写核心- my spring bean工厂（IoC、DI）

3.4 my spring bean工厂&#xff08;IoC、DI&#xff09; 加载类解析类获得类 3.4.0 分析 3.4.1 扫描指定包下的所有类 package com.czxy.bean;import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Objects;public class MyBeanFac…
阅读更多...
【GitHub项目推荐--十六进制编辑器】【转载】

【GitHub项目推荐--十六进制编辑器】【转载】

一款名为 ImHex 的十六进制编辑器获得了 15.3k 的 Star。十六进制编辑器可以让你以十六进制的形式查看或编辑文件的二进制数据&#xff0c;并用较为友好的界面来编辑二进制数据&#xff0c;和常见的十六进制编辑器 GNOME Hex Editor 等不一样&#xff0c;ImHex 功能非常强大&am…
阅读更多...
推荐文章
最新文章

Copyright @ 2022~2023