可在普通服务器上执行。完整代码分为数据生成、分块处理和批量写入三个核心模块:
using System;
using System.Collections.Concurrent;
using System.Data.SqlClient;
using System.Threading.Tasks;
using System.Diagnostics;
class BigDataProcessor
{
// 数据库连接字符串(需根据实际配置修改)
private const string ConnectionString = "Server=.;Database=BigDataDB;Integrated Security=true;";
// 分块大小(根据内存调整)
private const int BatchSize = 100000;
// 并行处理线程数(建议为CPU核心数2倍)
private const int MaxDegreeOfParallelism = 8;
static async Task Main(string[] args)
{
var processor = new BigDataProcessor();
await processor.CreateDatabaseStructure();
var sw = Stopwatch.StartNew();
await processor.ProcessBillionRecords();
sw.Stop();
Console.WriteLine($"总耗时: {
sw.Elapsed.TotalMinutes:N2} 分钟");
}
// 创建数据库结构
private async Task CreateDatabaseStructure(