select
a.COUNTY_FIPS
,COUNT(e.PROPERTY_ID) as house_count
,AVG(cast(e.AVM_FINAL_VALUE as bigint)) as avg_avm
,max(cast(e.AVM_FINAL_VALUE as bigint)) as max_avm
,min(cast(e.AVM_FINAL_VALUE as bigint)) as min_avm
from
RAW_Equity e
left join
(SELECT
SA_PROPERTY_ID, MM_FIPS_STATE_CODE, MM_FIPS_MUNI_CODE,
CASE
WHEN MM_FIPS_STATE_CODE < 10
THEN '0' + CAST(MM_FIPS_STATE_CODE as VARCHAR)
ELSE CAST(MM_FIPS_STATE_CODE as VARCHAR)
END
+ CASE
WHEN MM_FIPS_MUNI_CODE < 10
THEN '00' + CAST(MM_FIPS_MUNI_CODE as VARchar)
WHEN MM_FIPS_MUNI_CODE < 100
THEN '0' + CAST(MM_FIPS_MUNI_CODE as VARchar)
ELSE CAST(MM_FIPS_MUNI_CODE as VARchar)
END AS COUNTY_FIPS
FROM
RAW_Address) a ON a.SA_PROPERTY_ID = e.PROPERTY_ID
where
AVM_CONFIDENCE_SCORE >= 70
group by
a.COUNTY_FIPS
Is there any way I can improve the performance of this query? Schema for both the tables are shown below. I am was thinking about creating non clustered index on AVM_CONFIDENCE_SCORE but I think it will only going to increase the query time. Any help will be greatly appreciated.
RAWADDRESS table:
CREATE TABLE [dbo].[RAW_Address]
(
[SA_PROPERTY_ID] [int] NOT NULL,
[SA_SCM_ID] [int] NOT NULL,
[MM_STATE_CODE] [varchar](2) NOT NULL,
[MM_MUNI_NAME] [varchar](24) NOT NULL,
[MM_FIPS_STATE_CODE] [tinyint] NOT NULL,
[MM_FIPS_MUNI_CODE] [smallint] NOT NULL,
[MM_FIPS_COUNTY_NAME] [varchar](35) NOT NULL,
[SA_SITE_HOUSE_NBR] [varchar](20) NULL,
[SA_SITE_FRACTION] [varchar](10) NULL,
[SA_SITE_DIR] [varchar](2) NULL,
[SA_SITE_STREET_NAME] [varchar](40) NULL,
[SA_SITE_SUF] [varchar](4) NULL,
[SA_SITE_POST_DIR] [varchar](2) NULL,
[SA_SITE_UNIT_PRE] [varchar](10) NULL,
[SA_SITE_UNIT_VAL] [varchar](6) NULL,
[SA_SITE_CITY] [varchar](30) NULL,
[SA_SITE_STATE] [varchar](2) NOT NULL,
[SA_SITE_ZIP] [int] NULL,
[SA_SITE_PLUS_4] [smallint] NULL,
[SA_SITE_CRRT] [varchar](4) NULL,
[SA_MAIL_HOUSE_NBR] [varchar](20) NULL,
[SA_MAIL_FRACTION] [varchar](10) NULL,
[SA_MAIL_DIR] [varchar](2) NULL,
[SA_MAIL_STREET_NAME] [varchar](50) NULL,
[SA_MAIL_SUF] [varchar](4) NULL,
[SA_MAIL_POST_DIR] [varchar](2) NULL,
[SA_MAIL_UNIT_PRE] [varchar](10) NULL,
[SA_MAIL_UNIT_VAL] [varchar](6) NULL,
[SA_MAIL_CITY] [varchar](50) NULL,
[SA_MAIL_STATE] [varchar](2) NULL,
[SA_MAIL_ZIP] [int] NULL,
[SA_MAIL_PLUS_4] [smallint] NULL,
[SA_MAIL_CRRT] [varchar](4) NULL,
[SA_SITE_MAIL_SAME] [varchar](1) NULL
) ON [PRIMARY]
RAW Equity table:
CREATE TABLE [dbo].[RAW_Equity]
(
[PROPERTY_ID] [int] NOT NULL,
[SCM_ID] [int] NOT NULL,
[MM_STATE_CODE] [varchar](2) NOT NULL,
[MM_MUNI_NAME] [varchar](24) NOT NULL,
[MM_FIPS_STATE_CODE] [int] NOT NULL,
[MM_FIPS_MUNI_CODE] [int] NOT NULL,
[MM_FIPS_COUNTY_NAME] [varchar](35) NOT NULL,
[AVM_FINAL_VALUE] [int] NULL,
[AVM_LOW_VALUE] [int] NULL,
[AVM_HIGH_VALUE] [int] NULL,
[AVM_CONFIDENCE_SCORE] [int] NULL,
[FINAL_VALUE] [float] NULL,
[FIRST_POSITION_SR_UNIQUE_ID] [int] NULL,
[FIRST_POSITION_LOAN_DATE] [int] NULL,
[FIRST_POSITION_DOC_NBR] [varchar](20) NULL,
[FIRST_POSITION_LOAN_VAL] [int] NULL,
[FIRST_POSITION_LENDER_CODE] [int] NULL,
[FIRST_POSITION_LNDR_LAST_NAME] [varchar](50) NULL,
[FIRST_POSITION_LNDR_FIRST_NAME] [varchar](50) NULL,
[FIRST_POSITION_LENDER_TYPE] [varchar](1) NULL,
[FIRST_POSITION_LOAN_TYPE] [varchar](1) NULL,
[FIRST_POSITION_INTEREST_RATE_TYPE] [varchar](1) NULL,
[FIRST_POSITION_ESTIMATED_INTEREST_RATE] [float] NULL,
[FIRST_POSITION_LNDR_CREDIT_LINE] [varchar](1) NULL,
[FIRST_POSITION_MODELED_MORTGAGE_TYPE] [varchar](1) NULL,
[SECOND_POSITION_SR_UNIQUE_ID] [int] NULL,
[SECOND_POSITION_LOAN_DATE] [int] NULL,
[SECOND_POSITION_DOC_NBR] [varchar](20) NULL,
[SECOND_POSITION_LOAN_VAL] [int] NULL,
[SECOND_POSITION_LENDER_CODE] [int] NULL,
[SECOND_POSITION_LNDR_LAST_NAME] [varchar](50) NULL,
[SECOND_POSITION_LNDR_FIRST_NAME] [varchar](50) NULL,
[SECOND_POSITION_LENDER_TYPE] [varchar](1) NULL,
[SECOND_POSITION_LOAN_TYPE] [varchar](1) NULL,
[SECOND_POSITION_INTEREST_RATE_TYPE] [varchar](1) NULL,
[SECOND_POSITION_ESTIMATED_INTEREST_RATE] [float] NULL,
[SECOND_POSITION_LNDR_CREDIT_LINE] [varchar](1) NULL,
[SECOND_POSITION_MODELED_MORTGAGE_TYPE] [varchar](1) NULL,
[THIRD_POSITION_SR_UNIQUE_ID] [int] NULL,
[THIRD_POSITION_LOAN_DATE] [int] NULL,
[THIRD_POSITION_DOC_NBR] [varchar](20) NULL,
[THIRD_POSITION_LOAN_VAL] [int] NULL,
[THIRD_POSITION_LENDER_CODE] [int] NULL,
[THIRD_POSITION_LNDR_LAST_NAME] [varchar](50) NULL,
[THIRD_POSITION_LNDR_FIRST_NAME] [varchar](50) NULL,
[THIRD_POSITION_LENDER_TYPE] [varchar](1) NULL,
[THIRD_POSITION_LOAN_TYPE] [varchar](1) NULL,
[THIRD_POSITION_INTEREST_RATE_TYPE] [varchar](1) NULL,
[THIRD_POSITION_ESTIMATED_INTEREST_RATE] [float] NULL,
[THIRD_POSITION_LNDR_CREDIT_LINE] [varchar](1) NULL,
[THIRD_POSITION_MODELED_MORTGAGE_TYPE] [varchar](1) NULL,
[TOTAL_OUTSTANDING_LOANS] [bigint] NULL,
[LTV] [int] NULL,
[AVAILABLE_EQUITY] [int] NULL,
[LENDABLE_EQUITY] [int] NULL,
[PROCESS_ID] [int] NOT NULL,
[FILLER] [varchar](4) NULL,
CONSTRAINT [PK_RAW_Equity] PRIMARY KEY CLUSTERED
(
[PROPERTY_ID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
You could try this way :-
(willing to know, how much this one, helpful to you)
Declare #result Table
(
RowId Int Identity(1,1) Primary Key
,COUNTY_FIPS Varchar(100)
,MM_FIPS_STATE_CODE Int
,MM_FIPS_MUNI_CODE Int
,house_count Int
,avg_avm Int
,max_avm Int
,min_avm Int
)
Insert Into #result(MM_FIPS_STATE_CODE,MM_FIPS_MUNI_CODE,house_count,avg_avm,max_avm,min_avm)
Select a.MM_FIPS_STATE_CODE
,a.MM_FIPS_MUNI_CODE
,Count(e.PROPERTY_ID) as house_count
,Avg(Cast(e.AVM_FINAL_VALUE as bigint)) as avg_avm
,Max(Cast(e.AVM_FINAL_VALUE as bigint)) as max_avm
,Min(Cast(e.AVM_FINAL_VALUE as bigint)) as min_avm
From RAW_Equity As e With (Nolock)
Left Join RAW_Address As a With (Nolock) On e.PROPERTY_ID = a.SA_PROPERTY_ID
Where e.AVM_CONFIDENCE_SCORE >= 70
Group by a.MM_FIPS_STATE_CODE
,a.MM_FIPS_MUNI_CODE
Update r
Set r.COUNTY_FIPS = REPLICATE('0',2-LEN(RTRIM(r.MM_FIPS_STATE_CODE))) + RTRIM(r.MM_FIPS_STATE_CODE) + REPLICATE('0',3-LEN(RTRIM(r.MM_FIPS_MUNI_CODE))) + RTRIM(r.MM_FIPS_MUNI_CODE)
From #result As r
Select r.COUNTY_FIPS
,r.house_count
,r.avg_avm
,r.max_avm
,r.min_avm
From #result As r
1st try without any index, and after that create clustered index as mentioned and try AGAIN the above same query
CREATE INDEX IX_RAW_Address_SA_PROPERTY_ID ON RAW_Address(SA_PROPERTY_ID)
I would put an index on:
Table: RAW_Equity
Columns: PROPERTY_ID, AVM_CONFIDENCE_SCORE
and
Table: RAW_Address
Columns: SA_PROPERTY_ID
Include: MM_FIPS_STATE_CODE,MM_FIPS_MUNI_CODE
Your query can be simplified, which will likely make it faster:
select
REPLICATE('0',2-LEN(RTRIM(a.MM_FIPS_STATE_CODE)))
+ RTRIM(a.MM_FIPS_STATE_CODE)
+ REPLICATE('0',3-LEN(RTRIM(a.MM_FIPS_MUNI_CODE)))
+ RTRIM(a.MM_FIPS_MUNI_CODE)
AS COUNTY_FIPS
,COUNT(e.PROPERTY_ID) as house_count
,AVG(cast(e.AVM_FINAL_VALUE as bigint)) as avg_avm
,max(cast(e.AVM_FINAL_VALUE as bigint)) as max_avm
,min(cast(e.AVM_FINAL_VALUE as bigint)) as min_avm
from
RAW_Equity e
left join RAW_Address a
ON a.SA_PROPERTY_ID = e.PROPERTY_ID
where
e.AVM_CONFIDENCE_SCORE >= 70
group by
a.MM_FIPS_STATE_CODE, a.MM_FIPS_MUNI_CODE
If RAW_Address doesn't already have a CLUSTERED INDEX with SA_PROPERTY_ID as the first key of the index, then this may help:
CREATE INDEX IX_RAW_Address_SA_PROPERTY_ID ON RAW_Address(SA_PROPERTY_ID)
INCLUDE (MM_FIPS_STATE_CODE, MM_FIPS_MUNI_CODE)
Related
I have an issue with one of my MSSQL tables. The table has 1.2 Trillion rows about 1TB of data and growing. The table is partitioned into 8000 partitions, only about 800 are used. The others were created for expanded growth.
Inserts and selects are < 1s
My problem is Updates are very slow. To updated 1 record varchar(100) using the exact partition key and Identity Column key, it is 3s.
If I place the update code in a Stored Procedure, goes in less than 1s. If I add option (recompile), about 1s. Is there a way to fix this without adding recomile?
Thank you.
CREATE TABLE [dbo].[_tabletest](
[data_id] [bigint] IDENTITY(10000,1) NOT NULL,
[idx1_id] [smallint] NOT NULL,
[idx2_id] [bigint] NOT NULL,
[idx3_id] [bigint] NOT NULL,
[template_id] [bigint] NOT NULL,
[reference_id] [varchar](200) NOT NULL,
[data] [nvarchar](400) NULL,
[data_1] [tinyint] NULL,
[data_2] [varchar](max) NULL,
[data_3] [real] NULL,
[data_4] [tinyint] NULL,
[data_5] [tinyint] NULL,
[data_6] [bit] NULL,
[data_7] [nvarchar](50) NULL,
[data_8] [datetime] NULL,
[data_9] [nvarchar](50) NULL,
[data_10] [varchar](100) NULL,
[data_11] [varchar](100) NULL,
[data_12] [varchar](100) NULL,
[data_13] [varchar](300) NULL,
[data_14] [varchar](200) NULL,
[data_15] [uniqueidentifier] NULL,
[data_16] [varchar](600) NULL,
[data_17] [varchar](100) NULL,
[data_18] [varchar](100) NULL,
[data_19] [decimal](10, 5) NULL,
[data_20] [decimal](10, 5) NULL,
[data_21] [decimal](10, 5) NULL,
[data_22] [decimal](10, 5) NULL,
[data_23] [decimal](10, 5) NULL,
[data_24] [int] NULL,
[data_25] [int] NULL,
[data_26] [int] NULL,
[data_27] [int] NULL,
[data_28] [decimal](10, 5) NULL,
[data_29] [decimal](10, 5) NULL,
[data_30] [decimal](10, 5) NULL,
[data_31] [datetime] NULL,
[data_32] [decimal](10, 5) NULL,
[data_3] [bit] NULL,
[data_34] [varchar](max) NULL,
[data_35] [smallint] NULL,
[data_36] [bigint] NULL,
[data_37] [int] NULL,
[data_38] [real] NULL,
[data_39] [datetime] NULL,
[data_40] [varchar](2500) NULL,
CONSTRAINT [PK_data_id] PRIMARY KEY CLUSTERED
(
[idx1_id] ASC,
[idx2_id] ASC,
[idx3_id] ASC,
[data_id] ASC
)WITH (PAD_INDEX = ON, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, FILLFACTOR = 90)
)
GO
ALTER TABLE [dbo].[_tabletest] SET (LOCK_ESCALATION = AUTO)
GO
ALTER TABLE [dbo].[_tabletest] ADD CONSTRAINT [DF_1] DEFAULT ((0)) FOR [data_1]
GO
ALTER TABLE [dbo].[_tabletest] ADD CONSTRAINT [DF_2] DEFAULT (getutcdate()) FOR [data_39]
GO
ALTER TABLE [dbo].[_tabletest] ADD DEFAULT (newid()) FOR [data_15]
GO
UPDATE [_tabletest] SET [data_40] = 'test-data'
WHERE [idx1_id] = 1209 AND [idx2_id] = 113795 AND [idx3] = 41195716 AND [data_id] = 1329110156
;
CREATE TABLE [dbo].[TB_CUSTPARAM](
[CUSTTYPE] [varchar](10) NOT NULL,
[PARAMETER] [varchar](20) NULL,
[DATATYPE] [int] NULL,
[TEXT_PARAM] [varchar](50) NULL,
[RANGE_FROM] [int] NULL,
[RANGE_TO] [int] NULL,
[RATING] [int] NULL,
[DESCRIPTION] [varchar](50) NULL,
[TABLENAME] [varchar](50) NULL,
[COLUMNNAME] [varchar](100) NULL,
[SQLQUERY] [varchar](1000) NULL,
[SRNO] [int] NOT NULL,
[WEIGHT] [numeric](4, 0) NULL,
[RANGE_VIEW_NAME] [varchar](200) NULL
) ON [PRIMARY]
-------------------------------------------------------------------------
CREATE TABLE [dbo].[TEMP_CUSTPARAM](
[CUSTNO] [varchar](50) NOT NULL,
[RATING] [numeric](18, 0) NULL,
CONSTRAINT [PK_TEMP_CUSTPARAM] PRIMARY KEY CLUSTERED
(
[CUSTNO] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, FILLFACTOR = 90) ON [PRIMARY]
) ON [PRIMARY]
-------------------------------------------------------------------
CREATE TABLE [dbo].[TB_CUSTPERSONAL](
[CUSTNO] [varchar](50) NOT NULL,
[SEX] [varchar](30) NULL,
[MARITALSTATUS] [varchar](30) NULL,
[INCOMESOURCE] [int] NULL,
[OCCUPATION] [varchar](30) NULL,
[EMPLOYERSNAME1] [varchar](50) NULL,
[EMPLOYERPHNO1] [varchar](15) NULL,
[EMPLOYERADDR1] [varchar](100) NULL,
[YEARSOFSERVICE1] [numeric](30, 2) NULL,
[EMPLOYERSNAME2] [varchar](50) NULL,
[EMPLOYERPHNO2] [varchar](15) NULL,
[EMPLOYERADDR2] [varchar](100) NULL,
[YEARSOFSERVICE2] [numeric](30, 2) NULL,
[TOTALYEAROFSERVICE] [numeric](30, 2) NULL,
[NOOFDEPENDANTS] [varchar](10) NULL,
[CURRANNUALINCOME] [numeric](30, 3) NULL,
[QUALIFICATION] [varchar](30) NULL,
[DESIGNATION] [varchar](30) NULL,
[VERIFICATIONTYPE] [int] NULL,
[PURPOSEOFACCOUNT] [varchar](50) NULL,
[REFEREDBY] [varchar](50) NULL,
[ANTICIPATEDACTIVITY] [varchar](50) NULL,
[NI_NUMBER] [varchar](35) NULL,
[IDENTIFICATION_MARK] [varchar](50) NULL,
[PKID] [varchar](50) NULL,
[TXNTURNOVER] [varchar](50) NULL,
[MONTHLY_INCOME] [numeric](30, 3) NULL,
[FATHER_SPOUSE_NAME] [varchar](100) NULL,
[MOBILENO] [varchar](30) NULL,
[EMPLOYERTYPE] [varchar](30) NULL,
[PROFESSIONTYPE] [varchar](30) NULL,
[BUSINESSTYPE] [varchar](30) NULL,
[AREAOFOCCUPATION] [varchar](30) NULL,
[EXPTXNPERQTR] [varchar](30) NULL,
[TYPEOF4WHEELEROWN] [varchar](30) NULL,
[NATIONALITY] [varchar](50) NULL,
[MINORYN] [varchar](1) NULL,
[SRCITIZENYN] [varchar](1) NULL,
[FORM60_61YN] [varchar](1) NULL,
[PASSPORT_NO] [varchar](50) NULL,
[IF_SELF_EMPLOYED] [varchar](50) NULL,
[IF_BUSINESS] [varchar](50) NULL,
[SOURCE_OF_INCOME] [varchar](50) NULL,
[YOU_LIVE_IN] [varchar](50) NULL,
[YOUR_RESIDENCE_IS] [varchar](50) NULL,
[DI_DATE] [datetime] NULL,
[REC_UPDATEDT] [datetime] NULL,
[DAILY_INCOME] [varchar](50) NULL,
[EMPLOYERPHNO] [varchar](50) NULL,
[EMPLOYERADDR] [varchar](50) NULL,
[YEARSOFSERVICE] [varchar](50) NULL,
[FIRSTNAME] [varchar](50) NULL,
[MIDDLENAME] [varchar](50) NULL,
[LASTNAME] [varchar](50) NULL,
[PLACEOFBIRTH] [varchar](50) NULL,
[PASSPORT_YN] [varchar](50) NULL,
[EMIRATESID_YN] [varchar](50) NULL,
[RESIVISA_YN] [varchar](50) NULL,
[SALCERTIFICATE_YN] [varchar](50) NULL,
[EMPLOYMENT_STATUS] [varchar](50) NULL,
[EMPLOYERSNAME] [varchar](50) NULL,
[E_ADDRESS2] [varchar](50) NULL,
[E_ADDRESS3] [varchar](50) NULL,
[E_COUNTRY] [varchar](50) NULL,
[YEAROFSERVICE] [varchar](50) NULL,
[DEPENDENT_CHILDREN] [varchar](50) NULL,
[SPOUSE_EMP_STATUS] [varchar](50) NULL,
[ACCOMODATION] [varchar](50) NULL,
[PROFESSION] [varchar](50) NULL,
[RESVISANO] [varchar](50) NULL,
[RESVISAISSUEPLACE] [varchar](50) NULL,
[RESVISAISSUEDATE] [varchar](50) NULL,
[RESVISAEXPIRYDATE] [varchar](50) NULL,
[EMP_CODE] [varchar](50) NULL,
[PREV_EMPLOYER] [varchar](50) NULL,
[EMP_JOIN_DATE] [varchar](50) NULL,
[ACCOUNTOPERATEDBY] [varchar](50) NULL,
[RESIDENT_STATUS] [varchar](50) NULL,
[CREDIT_CARDNO] [varchar](50) NULL,
[ISSED_BY] [varchar](50) NULL,
[EXPIRY_DATE] [varchar](50) NULL,
[SCHEME_NAME] [varchar](50) NULL,
[TENURE_SCHEME] [varchar](50) NULL,
[INSTALLAMT_SCHEME] [varchar](50) NULL,
[NOOFINSTALL_SCHEME] [varchar](50) NULL,
[MATURITYAMT_SCHEME] [varchar](50) NULL,
[MONTHLYAMT_SCHEME] [varchar](50) NULL,
[TERM_DEPOSIT] [varchar](50) NULL,
[DATE_OF_ISSUE] [varchar](50) NULL,
[DATE_OF_MATURITY] [varchar](50) NULL,
[TERMDEPO_AMOUNT] [varchar](50) NULL,
[TERMDEPO_RENEWAL] [varchar](50) NULL,
[TERMDEPO_RATE] [varchar](50) NULL,
[INTIAL_DEPOSIT] [varchar](50) NULL,
[SPECIAL_INST] [varchar](50) NULL,
[NAME_CONTACTPERSON] [varchar](50) NULL,
[NAME_CONTACTNUMBER] [varchar](50) NULL,
[MODIFIED_ON] [varchar](50) NULL,
[MODIFIED_BY] [varchar](50) NULL,
[MOTHERSMAIDENNAME] [varchar](50) NULL,
[CUSTOMER_FIRSTNAME] [varchar](50) NULL,
[CUSTOMER_LASTNAME] [varchar](50) NULL,
[CUSTOMER_MIDDLENAME] [varchar](50) NULL,
[EMPLOYERCITY] [varchar](50) NULL,
[EMPLOYERSTATE] [varchar](50) NULL,
[EMPLOYERZIP] [varchar](50) NULL,
[RESIDENTCOUNTRY] [varchar](50) NULL,
[EMPLOYERCOUNTRYCODE] [varchar](50) NULL,
[INRODU_NAME] [varchar](50) NULL,
[RISKCATEGORY] [varchar](50) NULL,
[ENTITY_GROUP_CODE] [varchar](50) NULL,
[ACCTNO_INTRO] [varchar](50) NULL,
[ENTITY_CODE] [varchar](50) NULL,
CONSTRAINT [PK_TB_CUSTPERSONAL] PRIMARY KEY CLUSTERED
(
[CUSTNO] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, FILLFACTOR = 90) ON [PRIMARY]
) ON [PRIMARY]
----------------------------------------------------------
create
PROCEDURE PROC_CUSTRATING_INC
#WEIGHTAGE FLOAT,
#TAB_NAME VARCHAR(4000),
#COL_NAME VARCHAR(4000)
--#ENTITY_CODE VARCHAR(50),#ENTITY_GROUP_CODE VARCHAR(50)
AS
BEGIN
DECLARE #v_DAT_ DATETIME,
#v_DAT2 DATETIME,
#v_Pro_ VARCHAR(100),
#v_LAS_ DATETIME,
#v_SQL_ VARCHAR(500),
#v_COU_ FLOAT,
#v_TOT_ FLOAT,
#v_BAT_ FLOAT,
#v_SQLSTRING VARCHAR(1000)
SET NOCOUNT ON;
SET #v_DAT_=GETDATE();
SET #v_Pro_='PROC_CUSTRATING_INC';
DELETE FROM TEMP_CUSTPARAM;
SELECT #v_LAS_ = LASTRUNDT FROM TB_SCHEDULE WHERE PROCESS_ID ='2';
SET #v_SQLSTRING = 'INSERT INTO /*+ APPEND*/ TEMP_CUSTPARAM(CUSTNO, RATING)
SELECT DISTINCT Q.CUSTNO, ISNULL(((ISNULL(P.RATING,0) * '+ ISNULL(#WEIGHTAGE, '') +')/100),0) AS RATING
FROM '+ ISNULL(#TAB_NAME, '') +' Q INNER JOIN TB_CUSTPARAM P
ON Q.' + ISNULL(#COL_NAME, '') +'=P.TEXT_PARAM
AND P.TABLENAME='''+ ISNULL(#TAB_NAME, '') +''' AND P.COLUMNNAME='''+ ISNULL(#COL_NAME, '') +'''
AND TRUNC(Q.DI_DATE) >= '''+ ISNULL(#v_LAS_, '') +''' ';
PRINT 'v_SQLSTRING111=== ' + ISNULL(#v_SQL_, '');
EXECUTE (#v_SQLSTRING);
-- RETURN;
MERGE INTO TB_RATING_TEMP A
USING TEMP_CUSTPARAM E
ON (A.CUSTNO=E.CUSTNO)
WHEN MATCHED THEN
UPDATE SET
A.TEMP_RATING=ISNULL(E.RATING,0);
SET #v_SQLSTRING='UPDATE TB_RATING_TEMP A
SET
A.PARAMCOUNT=A.PARAMCOUNT + 1,
A.BRATING = NVL(A.BRATING,0) + NVL(A.TEMP_RATING,0)
WHERE
A.CUSTNO IN
(
SELECT Q.CUSTNO
FROM TB_CUSTPARAM P,'+ ISNULL(#TAB_NAME, '') +' Q
WHERE P.TEXT_PARAM=TO_CHAR(Q.'+ ISNULL(#COL_NAME, '') +')
)';
PRINT 'v_SQLSTRING222=== ' + ISNULL(#v_SQL_, '');
EXECUTE (#v_SQLSTRING);
PRINT 4;
SET #v_DAT2=GETDATE();
--PRINT 'PROCEDURE ' + ISNULL(#v_Pro_, '') + ' TAKES : ' + ISNULL(CONVERT(VARCHAR, trunc((((86400*(#v_DAT2-#v_DAT_))/60)/60)/24)), '') + ' Days ' + ISNULL(CONVERT(VARCHAR, trunc(((86400*(#v_DAT2-#v_DAT_))/60)/60)- 24*(trunc((((86400*(#v_DAT2-#v_DAT_))/60)/60)/24))), '') + ' Hrs. ' + ISNULL(CONVERT(VARCHAR, trunc((86400*(#v_DAT2-#v_DAT_))/60)- 60*(trunc(((86400*(#v_DAT2-#v_DAT_))/60)/60)) ), '') + ' Mins.' + ISNULL(CONVERT(VARCHAR, trunc(86400*(#v_DAT2-#v_DAT_))- 60*ISNULL((trunc((86400*(#v_DAT2-#v_DAT_))/60)), '') + ' Sec.'), '');
END;
Running
EXEC PROC_CUSTRATING_INC 11.0,'TB_CUSTPERSONAL','OCCUPATION'
results in the error
error converting data type varchar to float.
The problem I think is in the below statement
SET #v_SQLSTRING = 'INSERT INTO /*+ APPEND*/ TEMP_CUSTPARAM(CUSTNO, RATING)
SELECT DISTINCT Q.CUSTNO, ISNULL(((ISNULL(P.RATING,0) * '+ ISNULL(#WEIGHTAGE, '') +')/100),0) AS RATING
FROM '+ ISNULL(#TAB_NAME, '') +' Q INNER JOIN TB_CUSTPARAM P
ON Q.' + ISNULL(#COL_NAME, '') +'=P.TEXT_PARAM
AND P.TABLENAME='''+ ISNULL(#TAB_NAME, '') +''' AND P.COLUMNNAME='''+ ISNULL(#COL_NAME, '') +'''
AND TRUNC(Q.DI_DATE) >= '''+ ISNULL(#v_LAS_, '') +''' ';
The ISNULL(#WEIGHTAGE, '') should be changed to ISNULL(#WEIGHTAGE, 0.0) since #WEIGHTAGE is a float and you are trying to change a varchar '' to a float in this ISNULL statement which is causing it to fail.
We are trying to insert 20000 or more records using SqlBulkCopy.WriteToServer method, but it is taking 140 sec to insert into the table.
Is there any way we can improve the time using SqlBulkCopy as this is widely used for bulk insertion?
Calling code:
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(dConn))
{
bulkCopy.ColumnMappings.Add("SamplingFileId", "SamplingFileId");
/*... 74 more columns ...*/
bulkCopy.BulkCopyTimeout = 500; //Assign Destination Table Name
bulkCopy.DestinationTableName = "cvr_ReviewSamplingLoans";
bulkCopy.BatchSize = 5000;
bulkCopy.WriteToServer(dtLoansToUpload);
}
Sql Table Definition:
CREATE TABLE [dbo].[ReviewSamplingLoans](
[Id] [int] IDENTITY(1000,1) NOT NULL,
[SamplingFileId] [int] NOT NULL,
[ReviewId] [int] NOT NULL,
[LoanType] [nvarchar](10) NOT NULL,
[LoanNumber] [nvarchar](50) NOT NULL,
[BorrowerName] [nvarchar](150) NOT NULL,
[OriginalUPB] [money] NOT NULL,
[CurrentUPB] [money] NOT NULL,
[OriginalMonthlyPayment] [money] NULL,
[CurrentMonthlyP&I] [money] NULL,
[PaymentFrequency] [nvarchar](2) NULL,
[OriginalNoteRate] [float] NULL,
[CurrentInterestRate] [float] NULL,
[OriginationDate] [date] NULL,
[MaturityDate] [date] NULL,
[FirstPaymentDate] [date] NULL,
[InterestPaidThruDate] [date] NULL,
[NextPaymentDate] [date] NULL,
[DelinquencyStatus] [nvarchar](10) NULL,
[LatePaymentCount] [int] NULL,
[StatedTerm] [int] NULL,
[OriginalAmortizationTerm] [int] NULL,
[NoteType] [nvarchar](10) NULL,
[RateType] [nvarchar](10) NULL,
[LoanProductType] [nvarchar](10) NULL,
[BalloonDueDate] [date] NULL,
[BalloonPaymentAmount] [money] NULL,
[PropertyCity] [nvarchar](50) NULL,
[PropertyState] [nchar](2) NULL,
[PropertyZipcode] [nvarchar](10) NULL,
[PropertyType] [nvarchar](10) NULL,
[PropertyTypeDescription] [nvarchar](50) NULL,
[LienPosition] [tinyint] NULL,
[OriginalLTV] [float] NULL,
[CurrentLTV] [float] NULL,
[SeniorLienBalance] [money] NULL,
[OriginalCommitmentAmount] [money] NULL,
[CurrentCommitmentAmount] [money] NULL,
[OriginalCombinedLTV] [float] NULL,
[CurrentCombinedLTV] [float] NULL,
[OriginalPropertyValue] [money] NULL,
[CurrentPropertyValue] [money] NULL,
[AppraisalDate] [date] NULL,
[AppraisalType] [nvarchar](10) NULL,
[PMI] [nchar](1) NULL,
[LoanPurpose] [nchar](1) NULL,
[OccupancyType] [nchar](1) NULL,
[DocumentationType] [nvarchar](10) NULL,
[OriginalFICO] [smallint] NULL,
[CurrentFICO] [smallint] NULL,
[LastFICOUpdatedDate] [date] NULL,
[DebtToIncomeRatio] [float] NULL,
[InterestOnlyPeriod] [smallint] NULL,
[Modication] [nchar](1) NULL,
[Foreclosure] [nchar](1) NULL,
[ArmProductType] [nvarchar](10) NULL,
[ArmMargin] [float] NULL,
[ArmIndexType] [nvarchar](10) NULL,
[FirstRateAdjustmentDate] [date] NULL,
[NextRateAdjustmentDate] [date] NULL,
[LifeFloor] [float] NULL,
[LifeCeiling] [float] NULL,
[InitialRateCap] [float] NULL,
[PeriodicRateCap] [float] NULL,
[InternalRiskRating] [nvarchar](10) NULL,
[CurrentDebtServiceRatio] [float] NULL,
[CurrentNetOperIncome] [float] NULL,
[OriginalDebtServiceRatio] [float] NULL,
[OriginalNetOperIncome] [float] NULL,
[CurrentOccupancy] [float] NULL,
[PropertyTotalSquareFootage] [int] NULL,
[UnitCount] [smallint] NULL,
[PrepaymentPenalty] [nchar](1) NULL,
[PrepaymentPenaltyType] [int] NULL,
[PrepaymentPenaltyTerm] [smallint] NULL,
CONSTRAINT [PK_ReviewSamplingLoans] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
GO
ALTER TABLE [dbo].[ReviewSamplingLoans] WITH NOCHECK ADD CONSTRAINT [FK_ReviewSamplingLoans_ReviewCollateralTypes] FOREIGN KEY([LoanType])
REFERENCES [dbo].[ReviewSamplingLoans] ([Code])
GO
ALTER TABLE [dbo].[ReviewSamplingLoans] CHECK CONSTRAINT [FK_ReviewSamplingLoans_ReviewCollateralTypes]
GO
ALTER TABLE [dbo].[ReviewSamplingLoans] WITH NOCHECK ADD CONSTRAINT [FK_ReviewSamplingLoans_ReviewSamplingFiles] FOREIGN KEY([ReviewId])
REFERENCES [dbo].[Reviews] ([Id])
GO
ALTER TABLE [dbo].[ReviewSamplingLoans] CHECK CONSTRAINT [FK_ReviewSamplingLoans_ReviewSamplingFiles]
GO
Regarding Size, excel file contains 20000 rows which are added to datatable upon validation and using sqlbulkcopy inserting into table.
SqlBulkCopy can be performed faster when you enable the Tablelock on the options.
Have a look here SqlBulkCopyOptions Enumeration
But keep in mind that once the operation has started, no one will be able to use the table until it is finished. Microsoft says that this tenchnique improves the speed when inserting large amounts of data.
I have a sql statement that has been running fine for more than 6 months and recently it started taking an exorbitant time to execute.
DMIRecord contains 15,950 rows and InHouse_CSV_Backup has 149,226 rows. I set up an index on both MERSMin and MERSNUMBER on their respective tables and rebuilt the existing indexes but it has not helped.
I have another query that does the same thing but DMIRecord is a different source table called 'MERSRecord' and contains 28,841 rows and it runs fine. This query runs in 1 second and returns 2,048 rows
Can anyone give me some things to look at to resolve the problem. The last time I ran the below query it took 1 hour and 48 minutes to return 15,435 rows.
I know the join is not ideal but that is what we have to work with. There is nothing else that can connect these to externally imported data sources.
SELECT m.*
FROM [MERSReconciliation].[DMIRecord] m
LEFT JOIN [dbo].[InHouse_CSV_Backup] b ON CAST(LTRIM(RTRIM(m.[MERSMin])) AS VARCHAR(31)) = REPLACE(LTRIM(RTRIM(b.[MERSNUMBER)),
'-', '')
WHERE ( ( m.[Resolved] IS NULL
OR m.[Resolved] = 0
)
AND ( m.[Ignore Always] IS NULL
OR m.[Ignore Always] = 0
)
AND ( m.[Ignore] IS NULL
OR m.[Ignore] = 0
)
AND ( m.[Processed] IS NULL
OR m.[Processed] = 0
)
)
AND b.[MERSNUMBER] IS NULL
Sample Data for Join DMIRecord
100113800001196174
100113800001155030
100113800001204309
Sample Data for Join InHouse_CSV_Backup
1001138-0000008482-7
NULL
1001138-0000008479-3
Here are the Execution Plans for both Bad and Good
Bad: DMIRecords
Good: MERSRecord
I added * to the query to simplify the statement for posting. All needed columns are defined in the actual statement.
USE [ExtraNet]
GO
/****** Object: Table [MERSReconciliation].[DMIRecord] Script Date: 1/29/2015 5:01:35 PM ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_PADDING ON
GO
CREATE TABLE [MERSReconciliation].[DMIRecord](
[Id] [int] IDENTITY(1,1) NOT NULL,
[ClientNo] [varchar](300) NULL,
[DetailRecordCtlNo] [varchar](300) NULL,
[ReportingEntityNo] [varchar](300) NULL,
[OrganizationName] [varchar](300) NULL,
[MERSOrgId] [varchar](300) NULL,
[LienPriorityCode] [varchar](300) NULL,
[RealEstateSecurityCode] [varchar](300) NULL,
[FHACaseNo] [varchar](300) NULL,
[InvestLoanNo] [varchar](300) NULL,
[MERSMin] [varchar](300) NULL,
[LoanDate] [varchar](300) NULL,
[OrigMtgAmt] [varchar](300) NULL,
[EntityTypeQual] [varchar](300) NULL,
[NameTypeCode] [varchar](300) NULL,
[MtgrSSNo] [varchar](300) NULL,
[MtgrLastName] [varchar](300) NULL,
[MtgrFirstName] [varchar](300) NULL,
[MtgrMiddleName] [varchar](300) NULL,
[CoEntityTypeQual1] [varchar](300) NULL,
[CoNameTypeCode1] [varchar](300) NULL,
[CoMtgrSSNo1] [varchar](300) NULL,
[CoborrowerLastName1] [varchar](300) NULL,
[CoborrowerFirstName1] [varchar](300) NULL,
[CoborrowerMiddleName1] [varchar](300) NULL,
[CoEntityTypeQual2] [varchar](300) NULL,
[CoNameTypeCode2] [varchar](300) NULL,
[CoMtgrSSNo2] [varchar](300) NULL,
[CoborrowerLastName2] [varchar](300) NULL,
[CoborrowerFirstName2] [varchar](300) NULL,
[CoborrowerMiddleName2] [varchar](300) NULL,
[CoEntityTypeQual3] [varchar](300) NULL,
[CoNameTypeCode3] [varchar](300) NULL,
[CoMtgrSSNo3] [varchar](300) NULL,
[CoborrowerLastName3] [varchar](300) NULL,
[CoborrowerFirstName3] [varchar](300) NULL,
[CoborrowerMiddleName3] [varchar](300) NULL,
[CoEntityTypeQual4] [varchar](300) NULL,
[CoNameTypeCode4] [varchar](300) NULL,
[CoMtgrSSNo4] [varchar](300) NULL,
[CoborrowerLastName4] [varchar](300) NULL,
[CoborrowerFirstName4] [varchar](300) NULL,
[CoborrowerMiddleName4] [varchar](300) NULL,
[CoEntityTypeQual5] [varchar](300) NULL,
[CoNameTypeCode5] [varchar](300) NULL,
[CoMtgrSSNo5] [varchar](300) NULL,
[CoborrowerLastName5] [varchar](300) NULL,
[CoborrowerFirstName5] [varchar](300) NULL,
[CoborrowerMiddleName5] [varchar](300) NULL,
[CoEntityTypeQual6] [varchar](300) NULL,
[CoNameTypeCode6] [varchar](300) NULL,
[CoMtgrSSNo6] [varchar](300) NULL,
[CoborrowerLastName6] [varchar](300) NULL,
[CoborrowerFirstName6] [varchar](300) NULL,
[CoborrowerMiddleName6] [varchar](300) NULL,
[CoEntityTypeQual7] [varchar](300) NULL,
[CoNameTypeCode7] [varchar](300) NULL,
[CoMtgrSSNo7] [varchar](300) NULL,
[CoborrowerLastName7] [varchar](300) NULL,
[CoborrowerFirstName7] [varchar](300) NULL,
[CoborrowerMiddleName7] [varchar](300) NULL,
[CoEntityTypeQual8] [varchar](300) NULL,
[CoNameTypeCode8] [varchar](300) NULL,
[CoMtgrSSNo8] [varchar](300) NULL,
[CoborrowerLastName8] [varchar](300) NULL,
[CoborrowerFirstName8] [varchar](300) NULL,
[CoborrowerMiddleName8] [varchar](300) NULL,
[CoEntityTypeQual9] [varchar](300) NULL,
[CoNameTypeCode9] [varchar](300) NULL,
[CoMtgrSSNo9] [varchar](300) NULL,
[CoborrowerLastName9] [varchar](300) NULL,
[CoborrowerFirstName9] [varchar](300) NULL,
[CoborrowerMiddleName9] [varchar](300) NULL,
[CoEntityTypeQual10] [varchar](300) NULL,
[CoNameTypeCode10] [varchar](300) NULL,
[CoMtgrSSNo10] [varchar](300) NULL,
[CoborrowerLastName10] [varchar](300) NULL,
[CoborrowerFirstName10] [varchar](300) NULL,
[CoborrowerMiddleName10] [varchar](300) NULL,
[NuStreetNo] [varchar](300) NULL,
[NuStreetName] [varchar](300) NULL,
[NuStreetDir] [varchar](300) NULL,
[NuCityName] [varchar](300) NULL,
[NuStateAbbr] [varchar](300) NULL,
[NuPropZip] [varchar](300) NULL,
[NuPropSuffix] [varchar](300) NULL,
[NuPropUnitNo] [varchar](300) NULL,
[County] [varchar](300) NULL,
[MERsMOMflag] [varchar](300) NULL,
[PoolNo] [varchar](300) NULL,
[MERsFundingDate] [varchar](300) NULL,
[MERsOrigOrgId] [varchar](300) NULL,
[OrigNoteHldNm] [varchar](300) NULL,
[MinStatCd] [varchar](300) NULL,
[MERsPropertyPresOrgId] [varchar](300) NULL,
[InvMERsOrgId] [varchar](300) NULL,
[SecuritizationName] [varchar](300) NULL,
[PropertyUnitType] [varchar](300) NULL,
[DateAssignFrMERs] [varchar](300) NULL,
[MERsStatusReasonCode] [varchar](300) NULL,
[OccupancyStatus] [varchar](300) NULL,
[AgencyNbr] [varchar](300) NULL,
[SubservicingMERsOrgId] [varchar](300) NULL,
[Assignee] [varchar](300) NULL,
[AssigneeAddressInfo] [varchar](300) NULL,
[AssigneeCity] [varchar](300) NULL,
[AssigneeState] [varchar](300) NULL,
[AssigneeZipCode] [varchar](300) NULL,
[Assignor] [varchar](300) NULL,
[Resolved] [bit] NULL,
[Ignore Always] [bit] NULL,
[Ignore] [bit] NULL,
[Processed] [bit] NULL,
[ImportedDate] [datetime] NULL,
[ImportedBy] [varchar](50) NULL,
[UpdatedDate] [datetime] NULL,
[UpdatedBy] [varchar](50) NULL,
[F2FResolved] [bit] NULL,
[F2FIgnore Always] [bit] NULL,
[F2FIgnore] [bit] NULL,
[F2FProcessed] [bit] NULL,
[Comment] [varchar](max) NULL,
[F2FComment] [varchar](max) NULL,
CONSTRAINT [PK_DMIRecord] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
SET ANSI_PADDING OFF
GO
USE [ExtraNet]
GO
/****** Object: Table [dbo].[InHouse_CSV_Backup] Script Date: 1/29/2015 5:01:11 PM ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_PADDING ON
GO
CREATE TABLE [dbo].[InHouse_CSV_Backup](
[Loan_ID] [int] IDENTITY(1,1) NOT NULL,
[LOANNUMBER] [float] NULL,
[LOANNUMBE1] [float] NULL,
[MERSNUMBER] [nvarchar](31) NULL,
[LEADID] [float] NULL,
[CALCDATELE] [datetime] NULL,
[PBFIRSTNAM] [nvarchar](26) NULL,
[PBLASTNAME] [nvarchar](36) NULL,
[PBSOCIALSE] [nvarchar](51) NULL,
[PCFIRSTNAM] [nvarchar](26) NULL,
[PCLASTNAME] [nvarchar](36) NULL,
[PBVOICE] [nvarchar](13) NULL,
[PBWORKNUMB] [nvarchar](13) NULL,
[LOANSTATUS] [nvarchar](21) NULL,
[StatusDate] [nvarchar](50) NULL,
[StatusDays] [float] NULL,
[PBCREDITSC] [float] NULL,
[PCCREDITSC] [float] NULL,
[BDETOTAL11] [float] NULL,
[BDETOTAL21] [float] NULL,
[BDETOTAL10] [float] NULL,
[LTV] [float] NULL,
[COMBINEDLT] [float] NULL,
[HOUSINGRAT] [float] NULL,
[DEBTRATIO] [float] NULL,
[PBAGE] [float] NULL,
[PCAGE] [float] NULL,
[BDEYEARSAT] [float] NULL,
[BDEYEARSA1] [float] NULL,
[INVESTOR] [nvarchar](51) NULL,
[PRODUCTCOD] [nvarchar](21) NULL,
[PROGRAMNAM] [nvarchar](51) NULL,
[PROGRAMCAT] [nvarchar](16) NULL,
[LOANPURPOS] [nvarchar](26) NULL,
[DOCUMENTAT] [nvarchar](21) NULL,
[SPPURPOSEO] [nvarchar](51) NULL,
[LIENPOSITI] [nvarchar](16) NULL,
[FUNDINGTYP] [nvarchar](16) NULL,
[SPOCCUPANC] [nvarchar](26) NULL,
[LOANTYPE] [nvarchar](16) NULL,
[SPPROPERTY] [nvarchar](26) NULL,
[RATE] [float] NULL,
[LOANTERM] [float] NULL,
[LOANAMOUNT] [float] NULL,
[LBRANCHNUM] [nvarchar](11) NULL,
[LOEMPLOYEE] [nvarchar](16) NULL,
[LOFIRSTNAM] [nvarchar](26) NULL,
[LOLASTNAME] [nvarchar](36) NULL,
[LPFIRSTNAM] [nvarchar](26) NULL,
[LPLASTNAME] [nvarchar](36) NULL,
[LUFIRSTNAM] [nvarchar](26) NULL,
[LULASTNAME] [nvarchar](36) NULL,
[LEADSOURCE] [nvarchar](31) NULL,
[REFERRALSO] [nvarchar](51) NULL,
[BROKERCOMP] [float] NULL,
[BRCOMPANYN] [nvarchar](51) NULL,
[SELLERREAL] [float] NULL,
[SACOMPANYN] [nvarchar](51) NULL,
[SellerAgenPhone] [nvarchar](50) NULL,
[SABRANCHNA] [nvarchar](26) NULL,
[SAFIRSTNAM] [nvarchar](26) NULL,
[SALASTNAME] [nvarchar](36) NULL,
[ADDRESS] [nvarchar](36) NULL,
[SPCITY] [nvarchar](26) NULL,
[SPSTATE] [nvarchar](3) NULL,
[SPZIP] [nvarchar](11) NULL,
[SPCOUNTY] [nvarchar](21) NULL,
[PBPAADDRES] [nvarchar](36) NULL,
[PBPACITY] [nvarchar](26) NULL,
[PBPASTATE] [nvarchar](3) NULL,
[PBPAZIP] [nvarchar](11) NULL,
[CALCDATEPR] [datetime] NULL,
[CALCDATENO] [datetime] NULL,
[CALCDATEON] [datetime] NULL,
[CALCDATEP1] [datetime] NULL,
[CALCDATEAP] [datetime] NULL,
[CALCDATEP2] [datetime] NULL,
[CALCDATESU] [datetime] NULL,
[CALCDATEA1] [datetime] NULL,
[CALCDATEA2] [datetime] NULL,
[CALCDATEA3] [datetime] NULL,
[CLoanFunded] [datetime] NULL,
[AppraisalReceived] [datetime] NULL,
[CALCDATEES] [datetime] NULL,
[SHIPPED] [datetime] NULL,
[FIRSTPAYME] [datetime] NULL,
[LOCKDATE] [datetime] NULL,
[LOCKEXPIRE] [datetime] NULL,
[CALCDATEDO] [datetime] NULL,
[CALCDATED1] [datetime] NULL,
[CALCDATED2] [datetime] NULL,
[CALCDATEIN] [datetime] NULL,
[CALCDATEFU] [datetime] NULL,
[CLOSEDATE] [datetime] NULL,
[CANCELDATE] [datetime] NULL,
[CALCDATERE] [datetime] NULL,
[CALCDATEI1] [datetime] NULL,
[MONTHLYLOA] [float] NULL,
[TOTALMONTH] [float] NULL,
[ProductID] [float] NULL,
[CPI] [char](10) NULL,
[BrokeredOut] [datetime] NULL,
[Commitment] [varchar](25) NULL,
[Pool] [varchar](25) NULL,
[Points801] [float] NULL,
[Points802] [float] NULL,
[ConfirmedPrice] [float] NULL,
[InvPrice] [float] NULL,
[InvLockDate] [datetime] NULL,
[Units] [float] NULL,
[InvID] [float] NULL,
[Committed_Product] [varchar](50) NULL,
[FloodZone] [char](10) NULL,
[ModBack] [datetime] NULL,
[CommSplitID] [char](10) NULL,
[CommissionSplit] [varchar](50) NULL,
[PropertyClass] [varchar](25) NULL,
[PUDType] [varchar](50) NULL,
[InvBase] [float] NULL,
[LockConfBase] [float] NULL,
[DeedReceivedDate] [datetime] NULL,
[TitleReceivedDate] [datetime] NULL,
[InvestorClearedDate] [datetime] NULL,
[InvestorDeniedDate] [datetime] NULL,
[InvestorSuspendedDate] [datetime] NULL,
[FHACaseNum] [varchar](50) NULL,
[DeniedDate] [datetime] NULL,
[LockCancelled] [datetime] NULL,
[UWRecievedDate] [datetime] NULL,
[UWSuspended] [datetime] NULL,
[ForwardCancelled] [datetime] NULL,
[LeadB2B] [datetime] NULL,
[ServicingBegin] [datetime] NULL,
[ServicingEnd] [datetime] NULL,
[MICertNum] [varchar](50) NULL,
[WareHouseLine] [varchar](50) NULL,
[FirstCreditScoreDate] [datetime] NULL,
[DuplicateFlag] [varchar](50) NULL,
[MinCreditScore] [nchar](10) NULL,
[NMLS_ID] [nchar](10) NULL,
[PQ_ApprovedDate] [datetime] NULL,
[CompletedDate] [datetime] NULL,
[PQ_DeniedDate] [datetime] NULL,
[PQ_WithdrawnDate] [datetime] NULL,
[LeadInactiveDate] [datetime] NULL,
[LoanOriginatedDate] [datetime] NULL,
[IncometeAppDate] [datetime] NULL,
[HOEPAStatus] [nchar](10) NULL,
[FinanceMethod] [varchar](50) NULL,
[IOMonths] [nchar](10) NULL,
[MI_Type] [varchar](50) NULL,
[MI_Activated] [varchar](50) NULL,
[MI_ExpireDate] [datetime] NULL,
[Fee902] [money] NULL,
[Fee1002] [money] NULL,
[HOEPA] [nchar](10) NULL,
[RequiredCreditScore] [nchar](10) NULL,
[ApprovedWithdrawn] [datetime] NULL,
[Withdrawn] [datetime] NULL,
[BranchDenied] [datetime] NULL,
[PQSubmitted] [datetime] NULL,
[Prepay] [nchar](10) NULL,
[CountyCD] [varchar](3) NULL,
[StateCD] [varchar](2) NULL,
[IsTestLoan] [bit] NULL,
[ReqClose_Dt] [date] NULL,
[LockReneg_Dt] [date] NULL,
[PTD_Cond_Count] [int] NULL,
[AdminFee] [varchar](3) NULL,
[PricingTier] [varchar](12) NULL,
[Borr_MidName] [nvarchar](36) NULL,
[CoBorr2_FirstName] [nvarchar](26) NULL,
[CoBorr2_MidName] [nvarchar](36) NULL,
[CoBorr2_LastName] [nvarchar](36) NULL,
[CoBorr2_SSN] [varchar](15) NULL,
[CoBorr2_Address] [nvarchar](36) NULL,
[CoBorr2_City] [nvarchar](26) NULL,
[CoBorr2_State] [varchar](2) NULL,
[CoBorr2_ZIP] [varchar](10) NULL,
[CoBorr2_CredScore] [int] NULL,
[CoBorr3_FirstName] [nvarchar](26) NULL,
[CoBorr3_MidName] [nvarchar](36) NULL,
[CoBorr3_LastName] [nvarchar](36) NULL,
[CoBorr3_SSN] [varchar](15) NULL,
[CoBorr3_Address] [nvarchar](36) NULL,
[CoBorr3_City] [nvarchar](26) NULL,
[CoBorr3_State] [varchar](2) NULL,
[CoBorr3_ZIP] [varchar](10) NULL,
[CoBorr3_CredScore] [int] NULL,
[Created_Dt] [datetime] NULL CONSTRAINT [DF_InHouse_CSV_Backup_Created_Dt] DEFAULT (getdate()),
[LastUpdated_Dt] [datetime] NULL,
[IntentProceed_Dt] [date] NULL,
[ClearClose_Dt] [date] NULL,
[SellerRealtor_ID] [int] NULL,
[BuyerRealtor_ID] [int] NULL,
CONSTRAINT [PK_InHouse_CSV_Backup] PRIMARY KEY NONCLUSTERED
(
[Loan_ID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, FILLFACTOR = 90) ON [PRIMARY]
) ON [PRIMARY]
GO
SET ANSI_PADDING OFF
GO
I ran the tuning advisor and it didn't suggest anything to improve it.
Interesting thing to note. I went back to the original SQL 2008 server that hosted these before and both statements execute in under 3 seconds. Both execution plans are identical. On the SQL 2012 Server only the MERSRecord query's execution plan is the same. All three match the one listed above. Only DMIRecords on the SQL 2012 server is different. Possible Bug?
1/30/2012
Update showing solution from my question on dba.stackexchange.com
add OPTION (HASH JOIN) to the end of the statement forcing a hash join. Adding this resulted in the problem query executing in 1 second and returning the proper result set.
Use CTE to clean your data. Using functions in the JOIN condition will prevent the use of index.
;WITH CteTableA AS(
SELECT
TrimmedCol1 = CAST(LTRIM(RTRIM(m.[COL1])) AS VARCHAR(31)),
Resolved = ISNULL(Resolved, 0),
[Ignore Always] = ISNULL([Ignore Always], 0),
Ignore = ISNULL(Ignore, 0),
Processed = ISNULL(Processed, 0)
FROM TableA
)
,CteTableB AS(
SELECT
*,
TrimmedCol3 = REPLACE(LTRIM(RTRIM(b.[COL3])),'-', '')
FROM TableB
)
SELECT
a.*
FROM CteTableA a
LEFT JOIN CteTableB b
ON b.TrimmedCol3 = a.TrimmedCol1
WHERE
a.Resolved = 0
AND a.[Ignore Always] = 0
AND a.Ignore = 0
AND a.Processed = 0
AND b.ACCOUNTNUMBER IS NULL
EDIT:
Try cleaning up the data and insert it in a temp table. Then add necessary index.
CREATE TABLE #TempTableA(
TrimmedCol1 VARCHAR(31),
Resolved BIT,
[Ignore Always] BIT,
Ignore BIT,
Processed BIT
-- Other columns you might need
)
CREATE TABLE #TempTableB(
TrimmedCol3 VARCHAR(31),
MERSNUMBER NVARCHAR(31) NULL
-- Other columns you might need
)
INSERT INTO #TempTableA
SELECT
TrimmedCol1 = CAST(LTRIM(RTRIM(m.[COL1])) AS VARCHAR(31)),
Resolved = ISNULL(Resolved, 0),
[Ignore Always] = ISNULL([Ignore Always], 0),
Ignore = ISNULL(Ignore, 0),
Processed = ISNULL(Processed, 0)
FROM TableA
INSERT INTO #TempTableB
SELECT
TrimmedCol3 = REPLACE(LTRIM(RTRIM(b.[COL3])),'-', ''),
MERSNUMBER
FROM TableB
CREATE NONCLUSTERED INDEX #IX_TempTableA ON #TempTableA (TrimmedCol1)
CREATE NONCLUSTERED INDEX #IX_TempTableB ON #TempTableB (TrimmedCol3)
SELECT
a.*
FROM #TempTableA a
LEFT JOIN #TempTableB b
ON b.TrimmedCol3 = a.TrimmedCol1
WHERE
a.Resolved = 0
AND a.[Ignore Always] = 0
AND a.Ignore = 0
AND a.Processed = 0
AND b.MERSNUMBER IS NULL
I'm using SQL server 2008 Express R2 with Advanced Services. I'm a DBA in training, so I'm learning as I get more exposure to SQL. I have a Base table that needs to be split into a Header and Details table. The two tables will be linked via HeaderID. The Header will have one record and the Details can have up to 64 Name/Value records, associated with the one record in the Header table. I need a Query to copy the data from the Base table into the Header and Details tables. Here is the design:
Base Table:
CREATE TABLE [dbo].[tblBase](
[ID] [int] IDENTITY(1,1) NOT NULL,
[PartType] [varchar](50) NULL,
[SerialNumber] [varchar](50) NOT NULL,
[Station] [varchar](50) NULL,
[Reject] [varchar](50) NULL,
[Generation] [varchar](50) NULL,
[Mod_25_1_Orifice] [varchar](50) NULL,
[Mod_25_1_LowerBody] [varchar](50) NULL,
[Mod_25_1_OrificeHolderLot] [varchar](50) NULL,
[Mod_25_1_Future1] [varchar](50) NULL,
[Mod_25_1_Future2] [varchar](50) NULL,
[Mod_30_2_Force] [varchar](50) NULL,
[Mod_30_2_Dist] [varchar](50) NULL,
[Mod_35_1_Pintle] [varchar](50) NULL,
[Mod_35_1_Spring] [varchar](50) NULL,
[Mod_35_1_CenterPole] [varchar](50) NULL,
[Mod_35_1_LowerBodyOuterLot] [varchar](50) NULL,
[Mod_40_1_Force] [varchar](50) NULL,
[Mod_40_1_Dist] [varchar](50) NULL,
[Mod_40_2_Force] [varchar](50) NULL,
[Mod_40_2_Dist] [varchar](50) NULL,
[Mod_40_2_DistToPress] [varchar](50) NULL,
[Mod_40_2_OffsetDist] [varchar](50) NULL,
[Mod_50_1_LastForce] [varchar](50) NULL,
[Mod_50_1_LastDist] [varchar](50) NULL,
[Mod_50_3_LeakRate] [varchar](50) NULL,
[Mod_50_3_PressureDrop] [varchar](50) NULL,
[Mod_70_1_SmOring] [varchar](50) NULL,
[Mod_70_1_RestrOrifice] [varchar](50) NULL,
[Mod_70_1_UpperFlBrOring] [varchar](50) NULL,
[Mod_70_1_Filter] [varchar](50) NULL,
[Mod_70_1_FluidConn] [varchar](50) NULL,
[Mod_70_2_LgOring] [varchar](50) NULL,
[Mod_70_2_Coil] [varchar](50) NULL,
[Mod_70_2_Filter] [varchar](50) NULL,
[Mod_100_2_Current] [varchar](50) NULL,
[Mod_100_2_Dist] [varchar](50) NULL,
[Mod_105_1_InletTube] [varchar](50) NULL,
[Mod_105_1_Filter] [varchar](50) NULL,
[Mod_105_1_RestrOrifice] [varchar](50) NULL,
[Mod_105_1_PressDist] [varchar](50) NULL,
[Mod_105_1_TimeStamp] [datetime] NULL,
[Mod_110_1_BarrelFilter] [varchar](50) NULL,
[Mod_110_1_OrificePlate] [varchar](50) NULL,
[Mod_110_1_PressDist] [varchar](50) NULL,
[Mod_115_1_Force] [varchar](50) NULL,
[Mod_115_1_Dist] [varchar](50) NULL,
[Mod_115_1_Force2] [varchar](50) NULL,
[Mod_115_1_Dist2] [varchar](50) NULL,
[Mod_120_1_FluxBrAssem] [varchar](50) NULL,
[Mod_120_1_Coil] [varchar](50) NULL,
[Mod_120_1_LowerFluxBr] [varchar](50) NULL,
[Mod_120_1_PressDist] [varchar](50) NULL,
[Mod_130_1_Nest] [varchar](50) NULL,
[Mod_130_1_UnEnergizedFlow] [varchar](50) NULL,
[Mod_130_1_EnergizedFlow] [varchar](50) NULL,
[Mod_140_1_TimeStamp] [datetime] NULL,
[Mod_140_1_LeakRate] [varchar](50) NULL,
[Mod_140_1_PressureDrop] [varchar](50) NULL,
[Mod_35_1_TimeStamp] [datetime] NULL,
[Mod_70_1_TimeStamp] [datetime] NULL,
[Mod_130_1_TimeStamp] [datetime] NULL,
[Mod_135_1_TimeStamp] [datetime] NULL,
[Mod_135_2_TimeStamp] [datetime] NULL,
[Mod_150_1_LeakRate] [varchar](50) NULL,
[Mod_150_1_PressureDrop] [varchar](50) NULL,
[Mod_150_1_TimeStamp] [datetime] NULL,
[LoopA_pallet] [varchar](10) NULL,
[LoopB_pallet] [varchar](10) NULL,
[LoopC_pallet] [varchar](10) NULL,
CONSTRAINT [PK_tblLanco] PRIMARY KEY CLUSTERED
(
[SerialNumber] ASC
) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
Header Table:
CREATE TABLE [dbo].[tblHeader](
[HeaderID] [bigint] IDENTITY(1,1) NOT NULL,
[SerialNumber] [varchar](50) NOT NULL,
[PartType] [varchar](50) NULL,
[Station] [varchar](50) NULL,
[Reject] [varchar](50) NULL,
[Generation] [varchar](50) NULL,
CONSTRAINT [IX_tblLancoHeader] UNIQUE NONCLUSTERED
(
[SerialNumber] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
Details Table:
CREATE TABLE [dbo].[tblLDetail](
[DetailID] [bigint] IDENTITY(1,1) NOT NULL,
[HeaderID] [bigint] NOT NULL,
[Name] [varchar](100) NOT NULL,
[Value] [varchar](100) NULL
) ON [PRIMARY]
Hopefully, this is clear enough, such that I can get some help writing the query to copy the data out of the Base table, into the Header & Details table.
Thank You,
Rock
INSERT tblHeader (
[SerialNumber], [PartType], [Station], [Reject], [Generation]
)
SELECT [PartType], [SerialNumber], [Station], [Reject], [Generation]
FROM tblBase;
INSERT tblDetail (
[HeaderID], [Name], [VALUE]
)
SELECT h.HeaderID, 'Mod_25_1_Orifice', b.Mod_25_1_Orifice
FROM tblBase b
JOIN tblHeader h ON b.[SerialNumber] = h.[SerialNumber];
INSERT tblDetail (
[HeaderID], [Name], [VALUE]
)
SELECT h.HeaderID, 'Mod_25_1_LowerBody', b.Mod_25_1_LowerBody
FROM tblBase b
JOIN tblHeader h ON b.[SerialNumber] = h.[SerialNumber];
etc... insert to tblDetail 62 more times for the other 62 detail columns