Incremental batch delete and insert actions very slow in SQL Server - sql-server

Question

I have the following stored procedure which deletes and inserts rows in a table. It is running slow.
I have read various proposals and i have implemented:
Delete rows in batches and using derived table
Disable FK
Indexes on fields that are in the where clause.
So the way it should go is:
Tables have around 10 million records.
Every day i need to refresh around 15-30% of them. I use this SP to do it.
Source:
CREATE PROCEDURE [dbo].[spIncrementalUpdate]
-- Add the parameters for the stored procedure here
#table_inc nvarchar(30),
#table_target nvarchar(30),
#table_date nvarchar(30),
#field1 nvarchar(10),
#field2 nvarchar(10)
AS
BEGIN
-- SET NOCOUNT ON added to prevent extra result sets from
-- interfering with SELECT statements.
SET NOCOUNT ON;
DECLARE #logmessage as nvarchar(2048)
DECLARE #logdatacode as int
DECLARE #cmd as nvarchar(max)
DECLARE #fromjulian nvarchar(10)
DECLARE #datadate datetime
DECLARE #datadate_str nvarchar(10)
DECLARE #rows int
DECLARE #ParmDefinition nvarchar(500)
select #fromjulian = '114000'
print 'Using ' + #fromjulian
--
-- GET THE ROW COUNT OF THE INC TABLE.
--
IF #field2 = ''
BEGIN
SET #cmd = N'SELECT #retval = count(*) from ' +
#table_inc + ' where ' + #field1 +' > ' + #fromjulian
END
ELSE
BEGIN
SET #cmd = N'SELECT #retval = count(*) from ' +
#table_inc + ' where ' + #field1 +' > ' + #fromjulian +
' OR ' + #field2 +' > ' + #fromjulian
END
SET #ParmDefinition = N'#retval int OUTPUT'
EXEC sp_executesql #cmd, #ParmDefinition, #retval = #rows OUTPUT
--
if #rows <> 0
BEGIN
SET #logmessage = #table_inc + ' has ' +
cast(#rows as nvarchar(10)) +
' rows after ' + #fromjulian + ', deleting'
SET #logdatacode = 1000
--
-- Delete the records from original table based on #fromjulian
--
IF #field2 = ''
BEGIN
SET #cmd = N'delete ' + #table_target +
' from (select top(50000) * from ' + #table_target +
' where ' + #field1 + ' > ' + #fromjulian +
' order by ' + #field1 + ') ' +
#table_target
print #cmd
END
ELSE
BEGIN
SET #cmd = N'delete ' + #table_target +
' from (select top(50000) * from ' +
#table_target +
' where ' + #field1 + ' > ' + #fromjulian +
' OR ' + #field2 +' > ' + #fromjulian +
' order by ' + #field1 + ',' + #field2 + ') ' +
#table_target
print #cmd
END
SELECT 1
WHILE ##ROWCOUNT <> 0
BEGIN
EXEC sp_executesql #cmd
END
--
-- Inserting the records to target from INC table
--
IF #field2 = ''
BEGIN
SET #cmd = N'insert into ' + #table_target +
' select * from ' + #table_inc +
' where ' + #field1 +' > ' + cast(#fromjulian as nvarchar(10))
print #cmd
END
ELSE
BEGIN
SET #cmd = N'insert into ' + #table_target +
' select * from ' + #table_inc +
' where ' + #field1 +' > ' + cast(#fromjulian as nvarchar(10)) +
' OR ' + #field2 +' > ' + cast(#fromjulian as nvarchar(10))
print #cmd
END
print #cmd
EXEC sp_executesql #cmd
END
ELSE
BEGIN
SET #logmessage = 'NO ROWS IN ' + #table_inc + ' AFTER ' + cast(#fromjulian as nvarchar(10))
SET #logdatacode = 1001
END
--
-- LOG
--
INSERT INTO YLA_GROUP.[dbo].[sysssislog]
([event]
,[computer]
,[operator]
,[source]
,[sourceid]
,[executionid]
,[starttime]
,[endtime]
,[datacode]
,[databytes]
,[message])
VALUES
('spIncrementalUpdate','','','',NEWID(),NEWID(),getdate(),getdate(),#logdatacode,null,#logmessage)
print cast(#logdatacode as nvarchar(10)) + ' - ' + #logmessage
END

Answer 1

Since you are deleting rows in batches, you should remove the ORDER BY clause from your subquery because it is not necessary.
Here's the extract from your script:
IF #field2 = ''
BEGIN
SET #cmd = N'delete ' + #table_target +
' from (select top(50000) * from ' + #table_target +
' where ' + #field1 + ' > ' + #fromjulian + ') ' + #table_target
print #cmd
END
ELSE
BEGIN
SET #cmd = N'delete ' + #table_target +
' from (select top(50000) * from ' + #table_target +
' where ' + #field1 + ' > ' + #fromjulian +
' OR ' + #field2 +' > ' + #fromjulian + ') ' + #table_target
print #cmd
END
Remember that sorting can be an expensive operation, and even more expensive if you remove (correctly in this case) indexes.
Beyond that, there's not much you can do. Also consider that using dynamic SQL, execution plans are not cached. So, every time you execute the command inside #cmd the Query Engine needs to calculate the optimal execution plan. Unfortunately, as far as I can see, you need dynamic SQL.