Related
I have a staging table with more than 6 million rows of flattened data from a CSV file that I bulk inserted.
I need to take each rows, convert various column values from varchar to int/decimal/datetime, and input each row into numerous new database tables all with foreign key relationships, for now I'll simplify this to: (Parent, Child, OptionalChild) .
I don't need to read it row by row, as each single row contains the parent/child/optional child flattened data.
I am currently going through the records row by row using a SELECT TOP 1, then delete each row after its processed but this is taking hours obviously.
Would appreciate some faster / more efficient approaches.
DECLARE #Id UNIQUEIDENTIFIER;
DECLARE #Date DATETIME2;
DECLARE #Code VARCHAR(10);
DECLARE #Number INT;
DECLARE #OptionalChildCode VARCHAR(10);
DECLARE #OptionalChildNumber INT;
WHILE EXISTS(SELECT * FROM Records)
BEGIN
SELECT TOP 1
#Id = Id,
#Date = Date,
#Code = Code,
#Number = Number,
#OptionalChildCode = OptionalChildCode,
#OptionalChildNumber = OptionalChildNumber
FROM Records
DECLARE #OutputTable AS TABLE(Id UNIQUEIDENTIFIER, Name VARCHAR(10))
INSERT INTO (Parent) (ParentDate)
OUTPUT INSERTED.Id, 'ParentId' INTO #OutputTable(Id, Name)
VALUES (CONVERT(DATETIME2,#Date, 20))
INSERT INTO (Child)(ParentId, ChildCode, ChildNumber)
VALUES (
(SELECT ObjectId FROM #OutputTable WHERE Name = 'ParentId'),
#Code,
CONVERT(INT, #Number)
)
IF (#OptionalChildCode IS NULL)
BEGIN
INSERT INTO (Child)(ParentId, ChildCode, ChildNumber)
VALUES (
(SELECT ObjectId FROM #OutputTable WHERE Name = 'ParentId'),
#OptionalChildCode,
CONVERT(INT, #OptionalChildNumber)
)
END
DELETE FROM Records WHERE Id = #Id
END
Records table (all columns from CSV bulk import):
Columns: Id INT, Date VARCHAR(50), Code VARCHAR(50), Number VARCHAR(50), OptionalChildCode VARCHAR(50), OptionalChildNumber VARCHAR(50)
Target tables:
--Parent
Columns: (Id UNIQUEIDENTIFIER, ParentDate DATETIME2)
--Child
Columns: (Id UNIQUEIDENTIFIER, ParentId UNIQUEIDENTIFIER, ChildCode VARCHAR(10), ChildNumber INT)
Sample data (a row from Records table):
1, "2020-01-01-00-00", "Code123", "55", "OptionalCode456", "66"
Expected results:
--Row from Parent table:
111-222-333, 2020-01-01-00-00
--Row from Child table:
333-333-333, 111-222-333, "Code123", 55
--Row from Child table from optional child:
444-444-444, 111-222-333, "OptionalCode456", 66
The issue here is mainly that you need to get the inserted identity numbers matched against the original table, at the same time as inserting multiple child rows. You cannot use OUTPUT in an INSERT to output anything other than inserted columns.
We can start by using a hack involving MERGE to output the other columns.
We can then conditionally unpivot those rows in order to get one or two child rows to insert.
DECLARE #OutputTable AS TABLE(
Id UNIQUEIDENTIFIER,
Code VARCHAR(10),
Number INT,
OptionalChildCode VARCHAR(10),
OptionalChildNumber INT);
MERGE Parent p
USING Records r
ON 1 = 0 -- never match
WHEN NOT MATCHED THEN
INSERT (ParentDate)
VALUES (CONVERT(DATETIME2, r.[Date], 20))
OUTPUT inserted.Id, r.Code, CONVERT(INT, r.Number), OptionalChildCode, CONVERT(INT, r.OptionalChildNumber)
INTO #OutputTable (Id, Code, Number, OptionalChildCode, OptionalChildNumber)
;
INSERT INTO Child
(ParentId, ChildCode, ChildNumber)
SELECT t.Id, v.Code, v.Number
FROM #OutputTable t
CROSS APPLY (
SELECT t.Code, t.Number
UNION ALL
SELECT t.OptionalChildCode, t.OptionalChildNumber
WHERE OptionalChildCode IS NOT NULL
) v;
I am currently fetching an integer row, then converting the row to a Nvarchar in a hash table, then converting this to XML. The aim is to have all values returned in the one XML variable like so:
<item id ="001"/><item id ="002"/><item id ="003"/><item id ="004"/><item id ="005"/>
Currently my code returns this as an XML row like so:
col
---------------
<item id="60114" />
<item id="60116" />
<item id="60120" />
<item id="60122" />
<item id="60123" />
<item id="60124" />
<item id="60125" />
<item id="60129" />
Here is my code which i have anonimised:
DROP TABLE #ClientNumber
DECLARE #XMLClientID NVARCHAR (MAX)
CREATE TABLE #ClientNumber (ID int identity(1,1), [XMLClientID] NVARCHAR(20))
INSERT INTO #ClientNumber
SELECT '<item id ="'+ CAST([ClientId] AS NVARCHAR) + '"/>' AS [XMLClientID] FROM [dbo.].[MyView] WHERE Column = 'Condition' AND [ClientName] LIKE 'BLA%';
WITH xoutput AS (
SELECT CONVERT(xml, [XMLClientID]) AS col
FROM #ClientNumber)
SELECT *
FROM xoutput
Any steer would be great as the stuff will not work due to the for XML.
You could use integer values instead of convert and format it.
CREATE TABLE tbl (id int);
INSERT INTO tbl VALUES (60114), (60116), (60120), (60122)
GO
SELECT id
FROM tbl item
FOR XML AUTO
Output:
| XML_F52E2B61-18A1-11d1-B105-00805F49916B |
| :----------------------------------------------------------------------- |
| <item id="60114"/><item id="60116"/><item id="60120"/><item id="60122"/> |
dbfiddle here
Try this:
declare #table table (col varchar(100))
insert into #table values
('<item id="60114" />'),
('<item id="60116" />'),
('<item id="60120" />'),
('<item id="60122" />'),
('<item id="60123" />'),
('<item id="60124" />'),
('<item id="60125" />'),
('<item id="60129" />')
select cast(col as xml) from #table for xml path('')
Thanks to #John Cappelletti for the link. Here is my solution:
DECLARE #XMLClientID VARCHAR (MAX)
DECLARE #StringClientID VARCHAR (MAX)
DECLARE #XMLStringClient XML;
CREATE TABLE #ClientNumber (ID int identity(1,1), [XMLClientID] VARCHAR(20))
INSERT INTO #ClientNumber
--AMEND LINE BELOW TO SELECT TARGETS!!--
SELECT '<item id ="'+ CAST([ClientId] AS NVARCHAR) + '"/>' AS [XMLClientID] FROM [dbo.].[MyView] WHERE Column = 'Condition' AND [ClientName] LIKE 'BLA%';
SELECT DISTINCT [XMLClientID] = STUFF((Select '' +[XMLClientID]
FROM #ClientNumber
FOR XML Path(''),TYPE).value('(./text())[1]','varchar(max)'),1,0,'')
INTO #TempString
FROM #ClientNumber A
SET #StringClientID = (SELECT [XMLClientID] FROM #TempString)
SET #XMLStringClient = CAST(#StringClientID AS XML);
EXEC dbo.My_StoredProcedure #XMLStringClient
I have a table called #Tbl1, Each GROUP is 1 row and I have to extract the number of rows for each to #Tbl_Insert type.
Declare #Tbl1 Table (TableName NVARCHAR(250),ColumnName NVARCHAR(250),DataType NVARCHAR(250),DataValue NVARCHAR(250),InGroup NVARCHAR(250))
Declare #Tbl_Insert Table (ID INT, Name NVARCHAR(250), Age INT)
-- Sample Data
Insert Into #Tbl1 values ('#Tbl_Insert','ID','INT','1','Group1'),('#Tbl_Insert','Name','NVARCHAR(250)','John.Adam','Group1'),('#Tbl_Insert','Age','INT','10','Group1')
Insert Into #Tbl1 values ('#Tbl_Insert','ID','INT','2','Group2'),('#Tbl_Insert','Name','NVARCHAR(250)','Andy.Law','Group2'),('#Tbl_Insert','Age','INT','18','Group2')
I can convert #tbl1 to row by row into #Table_TEMP
Declare #Table_TEMP (Data nvarchar(max))
Insert Into #Table_TEMP
SELECT LEFT([DataValues] , LEN([DataValues] )-1)
FROM #Tbl1 AS extern
CROSS APPLY
(
SELECT Concat('''', Replace( ISNULL([DataValue],''), '''','' ) + ''',')
FROM #Tbl1 AS intern
WHERE extern.InGroup = intern.InGroup
Order By InGroup, ColumnName
FOR XML PATH('')
) pre_trimmed ( [DataValues])
GROUP BY InGroup, [DataValues]
I have to extract the number of rows in #Tbl1 ( Or #Table_TEMP) to #Tbl_Insert.
I don't want to use cursor to loop Insert row by row in #Table_TEMP, because, when you met with big data (example > 10000 rows). It's run to slow.
Please help.
I found sample in stackorverflow
Declare #tbl_Temp Table (Data NVARCHAR(MAX))
Declare #tbl2 Table (A NVARCHAR(MAX),B NVARCHAR(MAX),C NVARCHAR(MAX))
Insert Into #tbl_Temp values ('a1*b1*c1')
INSERT INTO #tbl2 (A,B,C)
SELECT PARSENAME(REPLACE(Data,'*','.'),3)
,PARSENAME(REPLACE(Data,'*','.'),2)
,PARSENAME(REPLACE(Data,'*','.'),1)
FROM #tbl_Temp
select * from #tbl2
It's nearly the same, but,
My data have "DOT", can not use PARSENAME
I must know numbers of DOT to Build Dynamics SQL??
PARSENAME only support 3 "DOT", It's null when More Dot.
EXAMPLE:
Declare #ObjectName nVarChar(1000)
Set #ObjectName = 'HeadOfficeSQL1.Northwind.dbo.Authors'
SELECT
PARSENAME(#ObjectName, 5) as Server4,
PARSENAME(#ObjectName, 4) as Server,
PARSENAME(#ObjectName, 3) as DB,
PARSENAME(#ObjectName, 2) as Owner,
PARSENAME(#ObjectName, 1) as Object
If, i understand correctly you will need to use apply in order to fetch the records & insert the data into other table
insert into #Tbl_Insert (ID, Name, Age)
select max(a.id) [id], max(a.Name) [Name], max(a.Age) [Age] from #Tbl1 t
cross apply
(values
(case when t.ColumnName = 'ID' then t.DataValue end,
case when t.ColumnName = 'Name' then t.DataValue end,
case when t.ColumnName = 'Age' then t.DataValue end, t.InGroup)
) as a(id, Name, Age, [Group])
group by a.[Group]
select * from #Tbl_Insert
I do both #Tbl_Insert & create 1 store to do like PARSENAME. It's improved performance.
create function dbo.fnGetCsvPart(#csv varchar(8000),#index tinyint, #last bit = 0)
returns varchar(4000)
as
/* function to retrieve 0 based "column" from csv string */
begin
declare #i int; set #i = 0
while 1 = 1
begin
if #index = 0
begin
if #last = 1 or charindex(',',#csv,#i+1) = 0
return substring(#csv,#i+1,len(#csv)-#i+1)
else
return substring(#csv,#i+1,charindex(',',#csv,#i+1)-#i-1)
end
select #index = #index-1, #i = charindex(',',#csv,#i+1)
if #i = 0 break
end
return null
end
GO
I want to sum up all the CountHours and display in pivoted column Total.Total is the sum of all SUnday , monday ... for particular UserName. How to achieve this
?
select FullName,Sunday,Monday,Tuesday,Wednesday,Thursday,Friday,Saturday,Total
from
(Select UserId_Fk,ISNULL(CAST(CountHours as decimal(18,2)),0)as CountHours,[Day] f rom CheckInCheckOut)
as convertedtable
inner join Users
on convertedtable.UserId_Fk=Users.UserId
PIVOT
(
SUM(CountHours)
FOR Day
IN([Sunday],[Monday],[Tuesday],[Wednesday],[Thursday],[Friday],[Saturday],[Total])
)
as PivotTable
Result of this query is:
Table Structure:
Table[CheckInCheckOut]
CheckInCheckOutId int
UserId_Fk int
CountHours nvarchar(50)
Day nvarchar(50)
Example would be appreciated.
you should calculate total column field, i.e it is not in list of pivot columns.
Data
create table #CheckInCheckOut(Id int identity(1,1),UserId_Fk int,CountHours varchar(50),[Day] varchar(50))
INSERT INTO #CheckInCheckOut(UserId_Fk,CountHours,[Day]) VALUES
(1,'2','Sunday'),(1,'2','Monday'),(1,'2','Tuesday'),(1,'2','Wednesday'),(1,'2','Thursday'),(1,'2','Friday'),(1,'2','Saturday')
,(2,'3','Sunday'),(2,'3','Monday'),(2,'3','Tuesday'),(2,'3','Wednesday'),(2,'3','Thursday'),(2,'3','Friday'),(2,'3','Saturday')
,(3,'3','Sunday'),(3,'3','Monday'),(3,'3','Tuesday'),(3,'3','Wednesday'),(3,'3','Thursday'),(3,'3','Friday'),(3,'3','Saturday')
create table #Users(UserId int identity(1,1),FullName varchar(50))
INSERT #Users(FullName) values('Abdul'),('khan'),('Tariq')
Query to find total too:
select FullName
,[Sunday] = SUM([Sunday])
,[Monday] = SUM([Monday])
,[Tuesday] = SUM([Tuesday])
,[Wednesday] = SUM([Wednesday])
,[Thursday] = SUM([Thursday])
,[Friday] = SUM([Friday])
,[Saturday] = SUM([Saturday])
, Total= SUM([Sunday]+[Monday]+[Tuesday]+[Wednesday]+[Thursday]+[Friday]+[Saturday])
from
(Select UserId_Fk,ISNULL(CAST(CountHours as decimal(18,2)),0)as CountHours,[Day]
from #CheckInCheckOut)
as convertedtable
inner join #Users
on convertedtable.UserId_Fk=#Users.UserId
PIVOT
(
SUM(CountHours)
FOR Day
IN([Sunday],[Monday],[Tuesday],[Wednesday],[Thursday],[Friday],[Saturday])
)
as PivotTable
GROUP BY FullName
Output
Also if u want total horizontal and vertical both then replace:
--GROUP BY FullName
GROUP BY ROLLUP(FullName);
For more follow link https://stackoverflow.com/a/17142530/1915855
DROP TABLE #CheckInCheckOut
DROP TABLE #Users
Try This Way here is the example.
Create a Table
CREATE TABLE cars
(
car_id tinyint,
attribute varchar(20),
value varchar(20),
sumd decimal(18,2)
)
Insert Values to it
insert into cars(car_id, attribute, value, sumd)
values (1, 'Make', 'VW',1),
(1, 'Model', 'Rabbit',2),
(1, 'Color', 'Gold',3),
(2, 'Make', 'Jeep',4),
(2, 'Model', 'Wrangler',5),
(2, 'Color', 'Gray',6)
For Making the Total
declare #Columns2 VARCHAR(8000)
declare #Sql VARCHAR(4000)
declare #Columns VARCHAR(8000)
SET #Columns = substring((select distinct ',['+attribute+']' from cars group by attribute for xml path('')),2,8000)
SET #Columns2 = substring((select distinct ',IsNull(['+attribute+'],0) as ['+attribute+']' from cars group by attribute for xml path('')),2,8000)
print #Columns
print #Columns2
SET #SQL = 'SELECT car_id, '+#Columns2+', total
FROM
(Select car_id,attribute, SUM(sumd) OVER (PARTITION BY attribute) as total
, sumd from cars) SourceData
PIVOT
(sum(sumd) for attribute in ('+#Columns+')) pivottable
Order by car_id '
exec(#sql)
My data looks like below:
<products>
<product ProductID="1" Price="79.99" Weight="30.00" Quantity="1">
<addon ProductAddonID="0" ControlTypeID="9" Price="25.00" Weight="0.00" Quantity="1" Name="yyy" Data="ASD" />
<addon ProductAddonID="89" ControlTypeID="0" Price="15.00" Weight="4.00" Quantity="1" Name="xxx" Data="" />
</product>
</products>
My SQL code looks like this:
INSERT INTO [Order].Items(OrderID, ProductID, Price, Weight, Quantity)
SELECT #OrderID, ProductID, Price, Weight, Quantity
FROM OPENXML (#XmlHandle, '/products/product',1)
WITH (ProductID INT '#ProductID',
Price DECIMAL(6,2) '#Price',
Weight DECIMAL(6,2) '#Weight',
Quantity INT '#Quantity')
SET #OrderItemId = SCOPE_IDENTITY()
INSERT INTO [Order].Addons(OrderItemID, ProductAddonID, ControlTypeID, Price, Weight, Quantity, [Name], DATA)
SELECT #OrderItemId, ProductAddonID, ControlTypeID, Price, Weight, Quantity, [Name], [Data]
FROM OPENXML(#XMLHandle, '/products/product/addon',1)
WITH (
ProductAddonID INT,
ControlTypeID INT,
Price DECIMAL(6,2),
Weight DECIMAL(6,2),
Quantity INT,
[Name] NVARCHAR(500),
[Data] NVARCHAR(max)
)
When I have multiple products/addons, all of the addons are inserting with the latest #OrderItemID ... I'm not sure how to work in my SQL that inserts the addon into the loop that iterates through the product nodes.
Could anyone point me in the right direction?
thanks in advance!
I think,
You need to insert records in Loop to get SCOPE_IDENTITY.
First put Order.Items datain temp table and then loop on it to insert in Order.Items table.
Following is the idea -- Not working code.
DECLARE #count INT
DECLARE #id INT
SET #count = 1
SET #id = totalNumberOfRecordsInTempTable -- Get records from xml to temp table first
WHILE #count <= #id
BEGIN
INSERT INTO YourTable (Column1, Column2, ...)
SELECT Column1, Column2, ... FROM SourceTable WHERE Id = #count
SET #count = #count + 1
SET #OrderItemId = SCOPE_IDENTITY()
INSERT INTO Order.AddOns
END
I have checked it and in loop you can get the SCOPE_IDENTITY.
declare #table table
(
id int,
quanity int
)
insert into #table select 1, 10
insert into #table select 2, 20
insert into #table select 3, 30
insert into #table select 4, 40
declare #table2 table
(
orderid int identity(1, 1),
quanity int
)
declare #id int
select #id = max(id) from #table
while #id > 0
begin
insert into #table2 (quanity)
select quanity from #table where id = #id
set #id = #id - 1
select SCOPE_IDENTITY()
end