SQL Server DELETE TOP slows down over time - sql-server

Question

I am running the following code to move data from a staging table (row-store) into production tables (column store). I observed that the process slows down over time. I suspect it could be because I am performing INSERT SELECT FROM, then DELETE until the table is empty. Could there be a better way to do this or What can I do to optimize this process? Should I rebuild the index in the middle? Appreciate your inputs.
Database
Partitioned tables by month
Simple recovery model
8 CPU, 16GB RAM
Table definition
No contraints, no triggers in the table
Average monthly rows in staging ~2 billion rows every when transfer
starts
CREATE TABLE [dbo].[AnalogT](
[SourceId] [int] NOT NULL,
[WindFarmId] [smallint] NOT NULL,
[StationId] [int] NOT NULL,
[TDIID] [int] NOT NULL,
[CTDIID] [int] NOT NULL,
[LogTime] [datetime2](0) NOT NULL,
[MeanValue] [real] NULL,
[MinValue] [real] NULL,
[MaxValue] [real] NULL,
[Stddev] [real] NULL,
[EndValue] [real] NULL,
[Published] [datetime2](3) NULL,
[TimeZoneOffset] [tinyint] NULL
) ON [Data]
GO
CREATE UNIQUE CLUSTERED INDEX [CIDX_AnalogT] ON [dbo].[AnalogT]
(
[LogTime] ASC,
[CTDIID] ASC,
[StationId] ASC,
[WindFarmId] ASC
) ON [Data]
GO
Get dataset to transfer
public class AnalogDeltaTransfer2ProductionService : DeltaTransfer2ProductionService
{
public override TdiType GetTargetTdiType()
{
return TdiType.Analog;
}
public override string GetCommandText(int batchSize)
{
string commandText = $"SELECT TOP {batchSize} " +
"[SourceId]," +
"[WindFarmId]" +
",[StationId]" +
",[TDIID]" +
",[CTDIID]" +
",[LogTime]" +
",[MeanValue]" +
",[MinValue]" +
",[MaxValue]" +
",[Stddev]" +
",[EndValue]" +
",[Published]" +
",[TimeZoneOffset]" +
",GETUTCDATE() [DateInsertedUtc]" +
$"FROM[dbo].[{GetStagingTable()}] ORDER BY LogTime, CTDIID, StationId, WindFarmId ";
return commandText;
}
public override string GetStagingTable()
{
var appSettings = new AppSettings();
return appSettings.deltasync_staging_analog;
}
public override string GetProductionTable()
{
var appSettings = new AppSettings();
return appSettings.deltasync_destination_analog;
}
}
Bulk load data
//Bulkd insert data from staging into production table
private void BulkTransferData(int batchSize, int batchNo)
{
var appSettings = new AppSettings();
var tdiTypeName = Enum.GetName(typeof (TdiType), GetTargetTdiType());
//source and destination databases is the same
var sourceAndDestinationConnectionString =
new StringBuilder(appSettings.deltasync_destination_connectionstring +
"Application Name=HPSDB DeltaSync DT2P {tdiType}")
.Replace("{tdiType}", tdiTypeName.ToUpper().Substring(0, 3))
.ToString();
using (var stagingConnection = new SqlConnection(sourceAndDestinationConnectionString))
{
stagingConnection.Open();
// get data from the source table as a stream datareader.
var commandSourceData = new SqlCommand(GetCommandText(batchSize), stagingConnection);
commandSourceData.CommandType = CommandType.Text;
commandSourceData.CommandTimeout = appSettings.deltasync_deltatransfer_prod_commandtimeout_secs.AsInt();
//prepare the fast reader
var reader = commandSourceData.ExecuteReader();
var transactionTimeOut =
TimeSpan.FromSeconds(appSettings.deltasync_deltatransfer_prod_trnxntimeout_secs.AsInt());
using (
var transactionScope = new TransactionScope(TransactionScopeOption.RequiresNew, transactionTimeOut))
{
//establish connection to destination production table
using (var destinationConnection = new SqlConnection(sourceAndDestinationConnectionString))
{
//grab connection from connection pool
destinationConnection.Open();
// set up the bulk copy object.
// note that the column positions in the source
// data reader match the column positions in the destination table so there is no need to map columns.
using (
var bulkCopy = new SqlBulkCopy(sourceAndDestinationConnectionString,
SqlBulkCopyOptions.TableLock))
{
try
{
// write from the source to the destination.
bulkCopy.DestinationTableName = GetProductionTable();
bulkCopy.BatchSize = appSettings.deltasync_deltatransfer_staging_batchsize.AsInt();
bulkCopy.BulkCopyTimeout =
appSettings.deltasync_deltatransfer_prod_commandtimeout_secs.AsInt();
bulkCopy.EnableStreaming = true;
bulkCopy.WriteToServer(reader);
}
catch (Exception ex)
{
//interrupt transaction and allow rollback process to commence
Log.Error(
$"DeltaTransfer2ProductionService.BulkTransferData/{tdiTypeName}/BulkInsert/{batchNo}: Faulted with with ff error: {ex}");
throw;
}
}
}
//establish connection to destination production table
using (var destinationConnection = new SqlConnection(sourceAndDestinationConnectionString))
{
//grab connection from connection pool
destinationConnection.Open();
//delete the rows that has been moved to destination table
try
{
//delete top x number of rows we just moved into prod
var selectTopXSqlText = GetCommandText(batchSize);
var sqlText = $"WITH cte AS({selectTopXSqlText}) DELETE FROM cte;";
var commandDropData = new SqlCommand(sqlText, destinationConnection);
commandDropData.CommandType = CommandType.Text;
commandDropData.CommandTimeout =
appSettings.deltasync_deltatransfer_prod_commandtimeout_secs.AsInt();
commandDropData.ExecuteNonQuery();
}
catch (Exception ex)
{
//interrupt transaction and allow rollback process to commence
Log.Error(
$"DeltaTransfer2ProductionService.BulkTransferData/{tdiTypeName}/DeleteTop/{batchNo}: Faulted with with ff error: {ex}");
throw;
}
}
//commit all changes
transactionScope.Complete();
transactionScope.Dispose();
}
}
}