Batch updating using loop on large table - sql-server

Question

I have two tables with overlaping data. One table is about 90% duplicate of the other. I need to identify the 10% of unique records in a table and move it to it's parent table. Both of these tables are 400 million + rows with 300+ columns. The method I am attempting is adding a flag field to uniquely ID the records I need to transfer however I need to update the field and am struggling with the logic. Below is what I have put together so far and it causes a never ending loop. There are no null values in either table.
Declare #counter int
Declare #RowsEffected int
Declare #RowsCnt int
Declare #Err int
SELECT #COUNTER = 1
SELECT #RowsEffected = 0
while (#counter > 0)
begin
set Rowcount 10000000
update Table1
set Existsflg = 1
where exists (
Select Fields
from Table1
Except
Select Fields
from table2 )
Select #RowsCnt = ##ROWCOUNT , #Err = ##ERROR
If #Err <> 0
begin
Print 'Problem Updating the records'
end
IF #RowsCnt = 0
SELECT #COUNTER = 0
ELSE
SELECT #RowsEffected = #RowsEffected + #RowsCnt
PRINT 'The total number of rows effected :'+convert(varchar,#RowsEffected)
WAITFOR DELAY '00:00:10'
END
SET ROWCOUNT 0
Go
Thanks!

Answer 1

This is how I did it one time.
I didn't use RowCount, I used Select TOP (N) and "while exists"
My "source" dbo.Employee table was on another server.
GO
USE [$(DestinationDatabaseName)]
GO
/*
READ ME !!!
Replace
$(SourceServer).$(SourceDatabaseName)
With (the Server and DatabaseName of the SOURCE data)
(ex:) [OtherServer].[OtherDatabase]
*/
--SubFolder: SQLReplicateReplacer
print '[uspEmployeeReplicateReplacer]'
go
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[uspEmployeeReplicateReplacer]') AND type in (N'P', N'PC'))
DROP PROCEDURE [dbo].[uspEmployeeReplicateReplacer]
Go
/*
declare #numberRowsAffected int
declare #ErrorNumber int
exec [dbo].[uspEmployeeReplicateReplacer] #numberRowsAffected output , #ErrorNumber output
print #numberRowsAffected
print #ErrorNumber
print ''
*/
CREATE PROCEDURE [dbo].[uspEmployeeReplicateReplacer] (
#numberRowsAffected int output --return
,
#ErrorNumber int output
)
AS
SET NOCOUNT ON
select #ErrorNumber = 0
declare #ErrorTracker int
declare #insertRowCount int
declare #updateRowCount int
select #insertRowCount = 0
select #updateRowCount = 0
IF OBJECT_ID('tempdb..#Employeeupdate') IS NOT NULL
begin
drop table #Employeeupdate
end
CREATE TABLE #Employeeupdate (
EmployeeKeyID int IDENTITY (1,1),
EmployeeUUID uniqueidentifier,
EmployeeLabel varchar(64),
EmployeeDescription varchar(128)
)
declare #ManualReplicationRowCount int
/* I put this value in a stored procedure, so I could change it in one place */
/* EXEC dbo.uspInternalSettingGetManualReplicationRowCount #ManualReplicationRowCount output */
Select #ManualReplicationRowCount = 1000
declare #MaximumLoopCounter int
select #MaximumLoopCounter = 10000
while (#MaximumLoopCounter > 0) and exists
(
Select
TOP 1 null
from [$(SourceServer)].[$(SourceDatabaseName)].dbo.Employee vart with (nolock)
where not exists
(
select null from dbo.Employee with (nolock) -- destinationTable
Where
/*
destinationTable.SOMEUNIQUECOLUMN1 = sourceTable.SOMEUNIQUECOLUMN1
and
destinationTable.SOMEUNIQUECOLUMN2 = sourceTable.SOMEUNIQUECOLUMN2
*/
dbo.Employee.EmployeeUUID = vart.EmployeeUUID
)
)
BEGIN
select #MaximumLoopCounter = #MaximumLoopCounter - 1
DELETE FROM #Employeeupdate
Insert into #Employeeupdate
(
EmployeeUUID,
EmployeeLabel,
EmployeeDescription
)
Select
TOP (#ManualReplicationRowCount)
EmployeeUUID,
EmployeeLabel,
EmployeeDescription
from [$(SourceServer)].[$(SourceDatabaseName)].dbo.Employee vart with (nolock)
where not exists
(
select null from dbo.Employee with (nolock) -- destinationTable
Where
/*
destinationTable.SOMEUNIQUECOLUMN1 = sourceTable.SOMEUNIQUECOLUMN1
and
destinationTable.SOMEUNIQUECOLUMN2 = sourceTable.SOMEUNIQUECOLUMN2
*/
dbo.Employee.EmployeeUUID = vart.EmployeeUUID
)
SET NOCOUNT OFF
Insert into dbo.Employee
(
EmployeeUUID,
EmployeeLabel,
EmployeeDescription
)
Select
EmployeeUUID,
EmployeeLabel,
EmployeeDescription
from
#Employeeupdate
SELECT #insertRowCount = ##ROWCOUNT , #ErrorTracker = ##ERROR
if #ErrorTracker <> 0
BEGIN
select #ErrorNumber = #ErrorTracker
select #MaximumLoopCounter = 0 --Bail Out !!!
END
SET NOCOUNT ON
END --End While Loop
/*
SET NOCOUNT OFF
Update dbo.Employee
Set
--EmployeeUUID = vart.EmployeeUUID,
EmployeeLabel = vart.EmployeeLabel,
EmployeeDescription = vart.EmployeeDescription
From
dbo.Employee with (nolock) , [$(SourceServer)].[$(SourceDatabaseName)].dbo.Employee vart with (nolock)
Where
--Relationship
dbo.Employee.EmployeeUUID = vart.EmployeeUUID
SELECT #updateRowCount = ##ROWCOUNT
SET NOCOUNT ON
*/
SELECT #numberRowsAffected = #insertRowCount + #updateRowCount
print '/#Employeeupdate COUNT/'
print #numberRowsAffected
print '-------------------------'
IF OBJECT_ID('tempdb..#Employeeupdate') IS NOT NULL
begin
drop table #Employeeupdate
end
SET NOCOUNT OFF
GO
GRANT EXECUTE ON dbo.uspEmployeeReplicateReplacer TO $(DBUSERNAME)
GO

Answer 2

I’d suggest you do this in batches of 1M-5M at a time because you have a ton of data to update.
What I would do in this case is:
a) add new column named Processed (bit) that would be updated for all rows that are processed
b) select 1M rows into temp table (this may not be needed but it will make things a bit
cleaner)
c) insert all non-duplicate records into other table
d) update rows and mark them as processes