SQL Server 2016 slow running procedure with attached plan - sql-server

Question

The stored procedure is running very slow in SQL Server 2016 as compared to SQL Server 2019. It's doing some common validation from a temporary staging table and inserting data in primary tables and mapping tables.
The mapping tables have clustered indexes on primary keys {Table1Id, Table2Id, Table3Id}. The procedure needs to process around 4 million rows. It finishes execution in 2 hours in SQL Server 2019, but takes probably 6-8 hours in SQL Server 2016 with the same data file.
Description of the stored proc -
There is a dbo.HouseholdTemp table with all rows to be processed its doing
Slice each row and insert one portion in dbo.Locations and another
potion in dbo.Individuals table if LocationGroupId or IndividualGroupId
does not exists or is empty.
Save the generated ID from those two tables and map them in dbo.Households table {LocationId, IndividualId}.
Select an existing MailingId from dbo.Mailings table with a JOIN query.
Insert another row in dbo.ListTypes table if the combination of Type and Code does not exists and get the ListTypes Id.
Insert a row in the dbo.HouseholdMailings table with MailingId,
LocationId, IndividualId , ListTypeId.
Execution plan is attached.
FYI, The SQL installation is on C drive where as the database files are on D drive. Please help!
Execution plan can be found here : https://www.brentozar.com/pastetheplan/?id=BybjNm5a8
CREATE PROCEDURE [dbo].[usp_ProcessHouseholdDatav2]
AS
BEGIN TRY
-- Declare variables
DECLARE #DataIngestionLogId int = 0;
DECLARE #HouseholdCount bigint = 0;
DECLARE #n int = 0;
DECLARE #householdrecords int = 0;
DECLARE #householdrecordsexistscount int = 0;
DECLARE #householdmailingsrecords int = 0;
DECLARE #householdmailingsrecordsexistscount int = 0;
DECLARE #listtyperecords int = 0;
DECLARE #tempid int = 0;
DECLARE #locationid int = 0;
DECLARE #individualid int = 0;
DECLARE #mailingid int = 0;
DECLARE #jobnumber nvarchar(6) = N'';
DECLARE #mergedpanelcode nvarchar(10) = N'';
DECLARE #locationgroupid nvarchar(10) = N'';
DECLARE #individualgroupid nvarchar(10) = N'';
DECLARE #listtype varchar(3) = '';
DECLARE #listcode varchar(4) = '';
DECLARE #findernumber nvarchar(50) = N'';
DECLARE #modelrank nvarchar(2) = N'';
DECLARE #listtypeid int = 0;
DECLARE #starttime datetime = GETDATE();
INSERT INTO dbo.DataIngestionLogs (FileName,
Message,
CreatedDate)
VALUES ('usp_ProcessHouseholdData', 'PROCESS STARTED ' + CAST(#HouseholdCount AS nvarchar(10)) + ' records to be processed', GETDATE());
SET #DataIngestionLogId = SCOPE_IDENTITY();
WHILE EXISTS (SELECT TOP 1 1 FROM dbo.HouseholdTemp ht WHERE ht.ProcessedDate IS NULL /*AND #n < 1000000*/ )
BEGIN
-- Retrieve Id, JobNumber and MergedPanelCode from HouseholdTemp table
SELECT TOP 1
#tempid = ht.Id,
#jobnumber = ht.JobNumber,
#mergedpanelcode = ht.MergedPanelCode,
#locationgroupid = ht.LocationGroupID,
#individualgroupid = ht.IndividualGroupID,
#listtype = ht.ListType,
#listcode = ht.ListCode,
#findernumber = ht.FinderNumber,
#modelrank = ht.ModelRank
FROM dbo.HouseholdTemp ht
WHERE ht.ProcessedDate IS NULL;
IF (#locationgroupid IS NULL OR #locationgroupid = '')
BEGIN
INSERT INTO dbo.Locations (LocationGroupId,
Address1,
Address2,
City,
State,
Zip10,
ZipCode,
DeliveryPointCode,
FIPSCounty,
CountyName,
AddressType,
PrimaryNumber,
PreDirectional,
StreetName,
StreetSuffix,
PostDirectional,
UnitDesignator,
UnitNumber,
PMBNumber,
CarrierRoute,
PenetrationCode,
MetroName,
CreatedDate,
ModifiedDate)
SELECT TOP 1
ht.LocationGroupID AS LocationGroupID,
ht.Address1 AS Address1,
ht.Address2 AS Address2,
ht.City AS City,
ht.State AS State,
ht.Zip10 AS Zip10,
ht.ZipCode AS ZipCode,
ht.DeliveryPointCode AS DeliveryPointCode,
ht.FIPSCounty AS FIPSCounty,
ht.CountyName AS CountyName,
ht.AddressType AS AddressType,
ht.PrimaryNumber AS PrimaryNumber,
ht.PreDirectional AS PreDirectional,
ht.StreetName AS StreetName,
ht.StreetSuffix AS StreetSuffix,
ht.PostDirectional AS PostDirectional,
ht.UnitDesignator AS UnitDesignator,
ht.UnitNumber AS UnitNumber,
ht.PMBNumber AS PMBNumber,
ht.CarrierRoute AS CarrierRoute,
ht.PenetrationCode AS PenetrationCode,
ht.MetroName AS MetroName,
GETDATE(),
GETDATE()
FROM dbo.HouseholdTemp ht
WHERE ht.Id = #tempid;
SET #locationid = SCOPE_IDENTITY();
END;
ELSE
BEGIN
-- Check if record exists with LocationGroupId in Locations table
IF (NOT EXISTS (SELECT TOP 1
1
FROM dbo.Locations l
WHERE l.LocationGroupId = RTRIM(LTRIM(#locationgroupid))))
BEGIN
-- PRINT 'Location does not exists with LocationGroupId : ' + #locationgroupid + '--> Insert in Locations table....'
INSERT INTO dbo.Locations (LocationGroupId,
Address1,
Address2,
City,
State,
Zip10,
ZipCode,
DeliveryPointCode,
FIPSCounty,
CountyName,
AddressType,
PrimaryNumber,
PreDirectional,
StreetName,
StreetSuffix,
PostDirectional,
UnitDesignator,
UnitNumber,
PMBNumber,
CarrierRoute,
PenetrationCode,
MetroName,
CreatedDate,
ModifiedDate)
SELECT TOP 1
ht.LocationGroupID AS LocationGroupID,
ht.Address1 AS Address1,
ht.Address2 AS Address2,
ht.City AS City,
ht.State AS State,
ht.Zip10 AS Zip10,
ht.ZipCode AS ZipCode,
ht.DeliveryPointCode AS DeliveryPointCode,
ht.FIPSCounty AS FIPSCounty,
ht.CountyName AS CountyName,
ht.AddressType AS AddressType,
ht.PrimaryNumber AS PrimaryNumber,
ht.PreDirectional AS PreDirectional,
ht.StreetName AS StreetName,
ht.StreetSuffix AS StreetSuffix,
ht.PostDirectional AS PostDirectional,
ht.UnitDesignator AS UnitDesignator,
ht.UnitNumber AS UnitNumber,
ht.PMBNumber AS PMBNumber,
ht.CarrierRoute AS CarrierRoute,
ht.PenetrationCode AS PenetrationCode,
ht.MetroName AS MetroName,
GETDATE(),
GETDATE()
FROM dbo.HouseholdTemp ht
WHERE ht.Id = #tempid;
SET #locationid = SCOPE_IDENTITY();
END;
ELSE
BEGIN
SELECT TOP 1
#locationid = Id
FROM dbo.Locations l
WHERE l.LocationGroupId = RTRIM(LTRIM((#locationgroupid)));
END;
END;
IF (#individualgroupid IS NULL OR #individualgroupid = '')
BEGIN
INSERT INTO dbo.Individuals (IndividualGroupId,
FirstName,
LastName,
DateOfBirth,
EmailAddress,
Gender,
MaritalStatus,
CreatedDate,
ModifiedDate)
SELECT ht.IndividualGroupID AS IndividualGroupId,
ht.FirstName AS FirstName,
ht.LastName AS LastName,
CAST(ht.DateOfBirth AS date) AS DateOfBirth,
ht.EmailAddress AS EmailAddress,
ht.Gender AS Gender,
ht.MaritalStatus AS MaritalStatus,
GETDATE(),
GETDATE()
FROM dbo.HouseholdTemp ht
WHERE ht.Id = #tempid;
SET #individualid = SCOPE_IDENTITY();
END;
ELSE
BEGIN
-- Check if record exists with IndividualGroupId in Individuals table
IF (NOT EXISTS (SELECT TOP 1
1
FROM dbo.Individuals i
WHERE i.IndividualGroupId = RTRIM(LTRIM((#individualgroupid)))))
BEGIN
INSERT INTO dbo.Individuals (IndividualGroupId,
FirstName,
LastName,
DateOfBirth,
EmailAddress,
Gender,
MaritalStatus,
CreatedDate,
ModifiedDate)
SELECT ht.IndividualGroupID AS IndividualGroupId,
ht.FirstName AS FirstName,
ht.LastName AS LastName,
CAST(ht.DateOfBirth AS date) AS DateOfBirth,
ht.EmailAddress AS EmailAddress,
ht.Gender AS Gender,
ht.MaritalStatus AS MaritalStatus,
GETDATE(),
GETDATE()
FROM dbo.HouseholdTemp ht
WHERE ht.Id = #tempid;
SET #individualid = SCOPE_IDENTITY();
END;
ELSE
BEGIN
SELECT TOP 1
#individualid = Id
FROM dbo.Individuals i
WHERE i.IndividualGroupId = RTRIM(LTRIM(#individualgroupid));
END;
END;
IF (#locationid > 0 AND #individualid > 0)
BEGIN
BEGIN TRY
INSERT INTO dbo.Households (LocationId,
IndividualId,
CreatedDate)
VALUES (#locationid, #individualid, GETDATE());
SET #householdrecords = #householdrecords + 1;
END TRY
BEGIN CATCH
END CATCH;
-- Get Mailing Id from JobNumber and MergedPanelCode
SELECT TOP 1
#mailingid = m.Id
FROM dbo.Mailings m
INNER JOIN dbo.Jobs j ON j.Id = m.JobId
INNER JOIN dbo.MergedPanels mp ON mp.Id = m.MergedPanelId
WHERE j.Number = #jobnumber
AND mp.Code = #mergedpanelcode;
IF (#mailingid > 0)
BEGIN
IF ((#listtype IS NOT NULL
OR #listtype != '')
AND (#listcode IS NOT NULL
OR #listcode != ''))
BEGIN
IF NOT EXISTS (SELECT TOP 1
1
FROM dbo.ListTypes lt
WHERE lt.Type = #listtype
AND lt.Code = #listcode)
BEGIN
INSERT INTO dbo.ListTypes (Type,
Code,
CreatedDate,
ModifiedDate)
VALUES (#listtype, #listcode, GETDATE(), GETDATE());
SET #listtypeid = SCOPE_IDENTITY();
SET #listtyperecords = #listtyperecords + 1;
END;
ELSE
BEGIN
SELECT TOP 1
#listtypeid = Id
FROM dbo.ListTypes lt
WHERE lt.Type = #listtype
AND lt.Code = #listcode;
END;
END;
IF (#listtypeid > 0)
BEGIN TRY
INSERT INTO dbo.HouseholdMailings (MailingId,
LocationId,
IndividualId,
ListTypeId,
FinderNumber,
ModelRank,
CreatedDate)
VALUES (#mailingid, #locationid, #individualid, #listtypeid, #findernumber, #modelrank, GETDATE());
SET #listtypeid = 0;
SET #householdmailingsrecords = #householdmailingsrecords + 1;
END TRY
BEGIN CATCH
END CATCH;
END;
ELSE
BEGIN
SET #householdmailingsrecordsexistscount = #householdmailingsrecordsexistscount + 1;
END;
END;
-- Update the ProcessedDate in HouseholdTemp table
UPDATE dbo.HouseholdTemp
SET ProcessedDate = GETDATE()
WHERE Id = #tempid;
SET #n = #n + 1;
SET #locationid = 0;
SET #individualid = 0;
SET #mailingid = 0;
END;
INSERT INTO dbo.DataIngestionLogs (FileName,
Message,
ParentId,
CreatedDate)
VALUES ('usp_ProcessHouseholdData', 'PROCESS COMPLETED ' + CAST(#n AS nvarchar(10)) + ' records processed.', #DataIngestionLogId, GETDATE());
END TRY
BEGIN CATCH
PRINT 'Error insert...';
INSERT INTO dbo.DataIngestionErrors
VALUES (SUSER_SNAME(), ERROR_NUMBER(), ERROR_STATE(), ERROR_SEVERITY(), ERROR_LINE(), ERROR_PROCEDURE(), ERROR_MESSAGE(), GETDATE(), #DataIngestionLogId);
END CATCH;