Related
One of my table column stores ~650,000 characters (each value of the column contains entire table). I know its bad design however, Client will not be able to change it.
I am tasked to convert the column into multiple columns.
I chose to use dbo.DelimitedSplit8K function
Unfortunately, it can only handle 8k characters at max.
So I decided to split the column into 81 8k batches using while loop and store the same in a variable table (temp or normal table made no improvement)
DECLARE #tab1 table ( serialnumber int, etext nvarchar(1000))
declare #scriptquan int = (select MAX(len (errortext)/8000) from mytable)
DECLARE #Counter INT
DECLARE #A bigint = 1
DECLARE #B bigint = 8000
SET #Counter=1
WHILE ( #Counter <= #scriptquan + 1)
BEGIN
insert into #tab1 select ItemNumber, Item from dbo.mytable cross apply dbo.DelimitedSplit8K(substring(errortext, #A, #B), CHAR(13)+CHAR(10))
SET #A = #A + 8000
SET #B = #B + 8000
SET #Counter = #Counter + 1
END
This followed by using below code
declare #tab2 table (Item nvarchar(max),itemnumber int, Colseq varchar(10)) -- declare table variable
;with cte as (
select [etext] ,ItemNumber, Item from #tab1 -- insert table name
cross apply dbo.DelimitedSplit8K(etext,' ')) -- insert table columns name that contains text
insert into #tab2 Select Item,itemnumber, 'a'+ cast (ItemNumber as varchar) colseq
from cte -- insert values to table variable
;WITH Tbl(item, colseq) AS(
select item, colseq from #tab2
),
CteRn AS(
SELECT item, colseq,
Rn = ROW_NUMBER() OVER(PARTITION BY colseq ORDER BY colseq)
FROM Tbl
)
SELECT
a1 Time,a2 Number,a3 Type,a4 Remarks
FROM CteRn r
PIVOT(
MAX(item)
FOR colseq IN(a1,a2,a3,a4)
)p
where a3 = 'error'
gives the desired output. However, just the loop takes 15 minutes to complete and overall query completes by 27 minutes. Is there any way I can make it faster? Total row count in my table is 2. So I don't think Index can help.
Client uses Azure SQL Database so I can't choose PowerShell or Python to accomplish this either.
Please let me know if more information is needed. I tried my best to mention everything I could.
Following is the stored procedure
ALTER PROCEDURE [dbo].[get_data_Dyna]
{
#param1 varchar(max) = null,
#param2 varchar(max) = null,
#start varchar(max) = null,
#end varchar(max) = null
}
AS
SELECT * from table where
(#param1 IS NULL OR column1 IN (SELECT data FROM dbo.delimited_list_to_table(#param1,',')))
AND (#param2 IS NULL OR column2 IN (SELECT data FROM dbo.delimited_list_to_table(#param2,',')))
AND ....?????
How this is working :
All the parameters can be comma seperated
#param1 value can be 'Germany' or 'Germany,USA' or null. This is working as expected.
Same goes for #param2
I'm trying to include rest of the parameters which is expected to work as follows :
#start='0' and #end='100' : In this case, where clause will look like this
...AND val BETWEEN #start AND #end
#start='48,60' and #end='51,99' : In this case, where clause will look like this
...AND ((val Between 48 and 51) or (val Between 60 and 99))
#start='48,60,75' and #end='51,99,203' : In this case, where clause will look like this
...AND ((val Between 48 and 51) or (val Between 60 and 99) or (val Between 75 and 203))
I'm unable to include above 2nd/3rd point correctly. I tried to write it dynamically which is working for single values [Point 1], but how to write point 2/3 ?
Any help is greatly appreciated.
Ok, i think the best approach here would be to use temp tables or table variable.
Lets go with temp tables.
create table #StartEnd (start int not null, end int not null, primary key (start,end))
then we insert from #start and #end into it using dbo.delimited_list_to_table. Now i am not sure about your implementation of it, so i will assume the values are numbered
insert into #StartEnd
select starts.data, ends.data
from dbo.delimited_list_to_table(#start,',') as starts
join dbo.delimited_list_to_table(#end,',') as ends
on starts.index = ends.index
Now we have to filter the values. Two approaches. Join or Exists condition
...
join #StartEnd on val between start and end
...
and exists (select 1 from #StartEnd where val between start and end)
Hope this helps
there you go. The comments / explainations are within the query
-- create a sample table
declare #tbl table
(
val int
)
-- put in some sample data
insert into #tbl
values (48), (60), (51), (99), (75), (203)
-- these are the input parameter
declare #start varchar(100),
#end varchar(100)
-- and these are the input value
select #start = '48,60,75',
#end = '51,99,203'
-- the actual query
; with
start_end as
(
-- here i am using [DelimitedSplit8K][1]
select s = s.Item, e = e.Item
from dbo.[DelimitedSplit8K](#start, ',') s
inner join dbo.[DelimitedSplit8K](#end, ',') e
on s.ItemNumber = e.ItemNumber
)
select t.val
from #tbl t
where exists
(
select *
from start_end x
where t.val between x.s and x.e
)
you can get it here DelimitedSplit8K
Sample input (from our understanding, we guess the your data):
select
* into ##delimit
from (
values
(1 ,'Ger','Ind', 100 )
,(2 ,'Ind',Null, 10 )
,(3 ,'Ger',Null, 24 )
,(4 ,'Ind','Ger', 54 )
,(5 ,'USA','Ind', 56 )
,(6 ,Null,'USA', 75 )-- NULL. But USA is three time came.
,(7 ,'USA','USA', 60 )-- same country with diff val.
,(8 ,'USA','USA', 80 )-- same country with diff val.
) demilit(Id,FromPr,ToPr,Val)
select * from ##delimit
Procedure (you just use this instead of your procedure):
CREATE PROCEDURE [dbo].[get_data_Dyna]
(#param1 varchar(max) = NULL,
#param2 varchar(max) = NULL,
#start varchar(max) = NULL,
#end varchar(max) = NULL)
AS
BEGIN
SELECT *
FROM ##delimit d
JOIN
( --| Here we check the val btw #start and #end
SELECT DISTINCT
s.FinalColumn StartVal, --|
e.FinalColumn EndVal --|
FROM
dbo.WithoutDelimit (#start, ',') s --| S means 'Start'
JOIN
(SELECT *
FROM dbo.WithoutDelimit (#end, ',')) e ON s.id = e.id --| E means 'End'
) se --| se mean StartEnd
ON d.val BETWEEN se.StartVal AND se.EndVal --| Here YOUR CONDITION is accomplished
WHERE
( -- | checks whether
frompr IN -- | column1 in #param1 or not
(SELECT FinalColumn FROM dbo.WithoutDelimit (#param1,',') -- | frompr means, 'column1'
) OR #param1 is NULL -- |
)
and ( -- | checks whether
ToPr in ( -- | column2 in #param2 or not
select FinalColumn from dbo.WithoutDelimit (#param2,',') -- | frompr means, 'column2'
) or #param2 is null -- |
)
end
Call stored procedure:
[get_data_Dyna] null,'usa','75','100,' -- 6 rows
[get_data_Dyna] 'Ind,Ger',null,'1,15','20,30' --2 and 3 rows are selected.
[get_data_Dyna] 'usa','usa','50,60','55,79'
-- 7 and 8 has same country. But due to Val, 8 has been rejected.
[get_data_Dyna] NULL,'usa','70,60','80,79'
-- 6 and 7 and 8 has been selected. Due to val condition.
Function (called from the stored procedure):
alter function WithoutDelimit -- We use one function for all conditions.
(#Parameter varchar (max),
#delimit varchar (1))
returns #FinalTable table (
Id int identity (1,1) -- Auto increment
, FinalColumn varchar (max) -- It returns the values as a column.
) as
begin
;with cte as -- recursive cte.
(
select convert (varchar (255), #Parameter + #delimit) con
, convert (varchar (255), #Parameter + #delimit) want
union all
select convert (varchar (255), stuff (con, 1, CHARINDEX (#demilit,con),'') )
, substring (con, 1, CHARINDEX (#delimit, con) - 1)
from cte
where con <> ''
) insert into #FinalTable (FinalColumn)
select want from cte
where con <> want
return
end
Revert us, if query need update.
I have a table called #Tbl1, Each GROUP is 1 row and I have to extract the number of rows for each to #Tbl_Insert type.
Declare #Tbl1 Table (TableName NVARCHAR(250),ColumnName NVARCHAR(250),DataType NVARCHAR(250),DataValue NVARCHAR(250),InGroup NVARCHAR(250))
Declare #Tbl_Insert Table (ID INT, Name NVARCHAR(250), Age INT)
-- Sample Data
Insert Into #Tbl1 values ('#Tbl_Insert','ID','INT','1','Group1'),('#Tbl_Insert','Name','NVARCHAR(250)','John.Adam','Group1'),('#Tbl_Insert','Age','INT','10','Group1')
Insert Into #Tbl1 values ('#Tbl_Insert','ID','INT','2','Group2'),('#Tbl_Insert','Name','NVARCHAR(250)','Andy.Law','Group2'),('#Tbl_Insert','Age','INT','18','Group2')
I can convert #tbl1 to row by row into #Table_TEMP
Declare #Table_TEMP (Data nvarchar(max))
Insert Into #Table_TEMP
SELECT LEFT([DataValues] , LEN([DataValues] )-1)
FROM #Tbl1 AS extern
CROSS APPLY
(
SELECT Concat('''', Replace( ISNULL([DataValue],''), '''','' ) + ''',')
FROM #Tbl1 AS intern
WHERE extern.InGroup = intern.InGroup
Order By InGroup, ColumnName
FOR XML PATH('')
) pre_trimmed ( [DataValues])
GROUP BY InGroup, [DataValues]
I have to extract the number of rows in #Tbl1 ( Or #Table_TEMP) to #Tbl_Insert.
I don't want to use cursor to loop Insert row by row in #Table_TEMP, because, when you met with big data (example > 10000 rows). It's run to slow.
Please help.
I found sample in stackorverflow
Declare #tbl_Temp Table (Data NVARCHAR(MAX))
Declare #tbl2 Table (A NVARCHAR(MAX),B NVARCHAR(MAX),C NVARCHAR(MAX))
Insert Into #tbl_Temp values ('a1*b1*c1')
INSERT INTO #tbl2 (A,B,C)
SELECT PARSENAME(REPLACE(Data,'*','.'),3)
,PARSENAME(REPLACE(Data,'*','.'),2)
,PARSENAME(REPLACE(Data,'*','.'),1)
FROM #tbl_Temp
select * from #tbl2
It's nearly the same, but,
My data have "DOT", can not use PARSENAME
I must know numbers of DOT to Build Dynamics SQL??
PARSENAME only support 3 "DOT", It's null when More Dot.
EXAMPLE:
Declare #ObjectName nVarChar(1000)
Set #ObjectName = 'HeadOfficeSQL1.Northwind.dbo.Authors'
SELECT
PARSENAME(#ObjectName, 5) as Server4,
PARSENAME(#ObjectName, 4) as Server,
PARSENAME(#ObjectName, 3) as DB,
PARSENAME(#ObjectName, 2) as Owner,
PARSENAME(#ObjectName, 1) as Object
If, i understand correctly you will need to use apply in order to fetch the records & insert the data into other table
insert into #Tbl_Insert (ID, Name, Age)
select max(a.id) [id], max(a.Name) [Name], max(a.Age) [Age] from #Tbl1 t
cross apply
(values
(case when t.ColumnName = 'ID' then t.DataValue end,
case when t.ColumnName = 'Name' then t.DataValue end,
case when t.ColumnName = 'Age' then t.DataValue end, t.InGroup)
) as a(id, Name, Age, [Group])
group by a.[Group]
select * from #Tbl_Insert
I do both #Tbl_Insert & create 1 store to do like PARSENAME. It's improved performance.
create function dbo.fnGetCsvPart(#csv varchar(8000),#index tinyint, #last bit = 0)
returns varchar(4000)
as
/* function to retrieve 0 based "column" from csv string */
begin
declare #i int; set #i = 0
while 1 = 1
begin
if #index = 0
begin
if #last = 1 or charindex(',',#csv,#i+1) = 0
return substring(#csv,#i+1,len(#csv)-#i+1)
else
return substring(#csv,#i+1,charindex(',',#csv,#i+1)-#i-1)
end
select #index = #index-1, #i = charindex(',',#csv,#i+1)
if #i = 0 break
end
return null
end
GO
I have a column in a table
Table 1
Tid. name fid
-----------------
1. Abc. 233
2. def. 344
3. xyz. 455
Table 2
did. Status. name FID
-------------------------------------------
1 Open. Abu,def,xyz 233,344,455
Now I want to split these fid and name from table2 in a stored procedure like this:
ID status name FID
---------------
1. Open. Abc. 233
2. Open. Def. 344
3 Open. xyz. 455
Update stored procedure
Query:
Create procedure as splitdata
As
Begin
Declare #fid varchar (500)
Select name, select item as fid
from spiltstring(#fid,','))
from table1
Inner join table2 on table1.fid = table2.fid
This shows error
Conversion failed when converting the varchar value '233,344,455' to data type int
In table1 fid is int and in table2 fid is varchar type.
Split string function is
CREATE FUNCTION SplitString
(
#Input NVARCHAR(MAX),
#Character CHAR(1)
)
RETURNS #Output TABLE (Item NVARCHAR(1000))
AS
BEGIN
DECLARE #StartIndex INT, #EndIndex INT
SET #StartIndex = 1
IF SUBSTRING(#Input, LEN(#Input) - 1, LEN(#Input)) <> #Character
BEGIN
SET #Input = #Input + #Character
END
WHILE CHARINDEX(#Character, #Input) > 0
BEGIN
SET #EndIndex = CHARINDEX(#Character, #Input)
INSERT INTO #Output(Item)
SELECT SUBSTRING(#Input, #StartIndex, #EndIndex - 1)
SET #Input = SUBSTRING(#Input, #EndIndex + 1, LEN(#Input))
END
RETURN
END
GO
How can I do this?
UPDATE
Ok according to #john solution I try this solution ...
But as I have multiple other tables
Like
Create procedure as sp2
#tick int
As
Select
Table 4.column, Table 5.column, Table 3.column,
--so here I used solution like this
(Select b.fid
from table2 t2
cross apply
(Select fid = s2.retval
from [udf-str-parase] (t2.fid, ',') s2) b),
Table 6.column
From
Table 2
Inner join
table4 on table4.id = table5.id
..........
Where
tick = #tick
When I try to execute only cross apply query this shows perfect result
But when I try to execute whole stored procedure like this
Exec sp2 28
I get this error:
Subquery returned more than 1 value.this is not permitted. When the subquery follows =,!=,<=,>=,>,< or when the subquery iIs used an expression.
Any solutions?
The the help of a split/parse function and a CROSS APPLY
Declare #Table2 table (did int,Status varchar(50),name varchar(50),FID varchar(50))
Insert Into #Table2 values
(1,'Open.','Abu,def,xyz','233,344,455')
Select ID = Seq
,A.Status
,B.Name
,B.FID
From #Table2 A
Cross Apply (
Select Name=S1.RetVal
,FID =S2.RetVal
,Seq =S1.RetSeq
From [dbo].[udf-Str-Parse](A.Name,',') S1
Join [dbo].[udf-Str-Parse](A.FID,',') S2
on S1.RetSeq=S2.RetSeq
) B
Returns
ID Status Name FID
1 Open. Abu 233
2 Open. def 344
3 Open. xyz 455
The Split/Parse UDF if needed
CREATE FUNCTION [dbo].[udf-Str-Parse] (#String varchar(max),#Delimiter varchar(10))
Returns Table
As
Return (
Select RetSeq = Row_Number() over (Order By (Select null))
,RetVal = LTrim(RTrim(B.i.value('(./text())[1]', 'varchar(max)')))
From (Select x = Cast('<x>'+ replace((Select #String as [*] For XML Path('')),#Delimiter,'</x><x>')+'</x>' as xml).query('.')) as A
Cross Apply x.nodes('x') AS B(i)
);
--Select * from [dbo].[udf-Str-Parse]('Dog,Cat,House,Car',',')
--Select * from [dbo].[udf-Str-Parse]('John Cappelletti was here',' ')
--Select * from [dbo].[udf-Str-Parse]('this,is,<test>,for,< & >',',')
given this table and data:
DECLARE #Table table (RowID int, RowCode char(1), RowValue int);set nocount on
INSERT #Table VALUES ( 6,'A',3757 )
INSERT #Table VALUES ( 5,'A',37827)
INSERT #Table VALUES (14,'A',48411)
INSERT #Table VALUES ( 1,'A',48386)
INSERT #Table VALUES (20,'A',48450)
INSERT #Table VALUES ( 7,'A',46155)
INSERT #Table VALUES (13,'A',721 )
INSERT #Table VALUES ( 2,'A',49335)
INSERT #Table VALUES (15,'A',4700 )
INSERT #Table VALUES (19,'A',64416)
INSERT #Table VALUES ( 8,'A',27246)
INSERT #Table VALUES (12,'B',54929)
INSERT #Table VALUES (16,'B',3872 )
INSERT #Table VALUES ( 3,'C',728 )
INSERT #Table VALUES (11,'C',1050 )
INSERT #Table VALUES ( 9,'C',3191 )
INSERT #Table VALUES (17,'C',866 )
INSERT #Table VALUES ( 4,'C',838 )
INSERT #Table VALUES (10,'D',550 )
INSERT #Table VALUES (18,'D',1434 );set nocount off
I need this:
VVVVVVVV
RowID RowCode RowValue RowChunk
----- ------- -------- --------
1 A 48386 1
2 A 49335 1
5 A 37827 1
6 A 3757 1
7 A 46155 1
8 A 27246 2
13 A 721 2
14 A 48411 2
15 A 4700 2
19 A 64416 2
20 A 48450 3
12 B 54929 4
16 B 3872 4
3 C 728 5
4 C 838 5
9 C 3191 5
11 C 1050 5
17 C 866 5
10 D 550 6
18 D 1434 6
RowChunk starts at 1 and is incremented by 1 for each RowCode change and/or when there have been 5 of the same RowCode values.
Basically my solution uses the same approach as yours, only with slightly different devices employed.
WITH NumberedRows AS (
SELECT
RowID,
RowCode,
RowValue,
CodeChunk = (ROW_NUMBER() OVER (PARTITION BY RowCode ORDER BY RowID) - 1) / 5
FROM #Table
)
SELECT
RowID,
RowCode,
RowValue,
RowChunk = DENSE_RANK() OVER (ORDER BY RowCode, CodeChunk)
FROM NumberedRows
I don't think there's an analysis function, or any reasonable combination of such, which will address this. You'll have to do it RBAR with a cursor or, slightly faster in my experience, a loop.
This example of looping assumes that RowID is unique. If RowID is not the clustered PK, this will be very slow, so if that's the case you'll want to create a temp table.
DECLARE #RowID INT = (SELECT MIN(RowID) FROM #Table)
DECLARE #MaxRowID INT = (SELECT MAX(RowID) FROM #Table)
DECLARE #RowCode CHAR(1)
DECLARE #LastRowCode CHAR(1)
DECLARE #RowValue INT
DECLARE #Chunk INT = 0
DECLARE #RecsThisChunk INT
DECLARE #Results TABLE (RowID INT NOT NULL PRIMARY KEY, RowCode CHAR(1) NOT NULL, RowValue INT NOT NULL, Chunk INT NOT NULL)
WHILE #RowID <= #MaxRowID
BEGIN
-- Handle gaps in RowID
IF NOT EXISTS (SELECT * FROM #Table WHERE RowID = #RowID) GOTO EndOfLoop
-- Load values for this record
SELECT #RowCode = RowCode, #RowValue = RowValue FROM #Table WHERE RowID = #RowID
IF #LastRowCode IS NULL OR #RowCode <> #LastRowCode OR #RecsThisChunk = 5
BEGIN
-- Start a new chunk
SET #Chunk = #Chunk + 1
SET #RecsThisChunk = 1
END
ELSE
BEGIN
-- Same chunk
SET #RecsThisChunk = #RecsThisChunk + 1
END
SET #LastRowCode = #RowCode
INSERT INTO #Results (RowID, RowCode, RowValue, Chunk) VALUES (#RowID, #RowCode, #RowValue, #Chunk)
EndOfLoop:
SET #RowID = #RowID + 1
END
SELECT * FROM #Results
You may have tweak this a bit for 2005, I use 2008 routinely and don't recall all the little differences.
FYI, the results you show don't quite match the sample data.
Hope this helps! The only alternative I see is a cursor, or handling this in the application layer.
this does the trick without a loop:
;WITH NumberedRows AS (
SELECT
r.RowID, r.RowCode, r.RowValue, CEILING(ROW_NUMBER() OVER(PARTITION BY r.RowCode ORDER BY r.RowCode,r.RowID)/5.0) AS CodeRowChunk
FROM #Table r
)
, AllChunks AS (
SELECT r.*,ROW_NUMBER() OVER(ORDER BY RowCode,CodeRowChunk) AS ChunkRowNumber
FROM (SELECT DISTINCT
RowCode, CodeRowChunk
FROM NumberedRows) r
)
SELECT
a.RowID, RowCode, a.RowValue,
(SELECT ChunkRowNumber FROM AllChunks c WHERE c.RowCode=a.RowCode and c.CodeRowChunk=a.CodeRowChunk) AS RowChunk
FROM NumberedRows a
This is the answer you are looking for :
create Table [table] (RowID int, RowCode char(1), RowValue int)
INSERT [Table] VALUES ( 6,'A',3757 )
INSERT [Table] VALUES ( 5,'A',37827)
INSERT [Table] VALUES (14,'A',48411)
INSERT [Table] VALUES ( 1,'A',48386)
INSERT [Table] VALUES (20,'A',48450)
INSERT [Table] VALUES ( 7,'A',46155)
INSERT [Table] VALUES (13,'A',721 )
INSERT [Table] VALUES ( 2,'A',49335)
INSERT [Table] VALUES (15,'A',4700 )
INSERT [Table] VALUES (19,'A',64416)
INSERT [Table] VALUES ( 8,'A',27246)
INSERT [Table] VALUES (12,'B',54929)
INSERT [Table] VALUES (16,'B',3872 )
INSERT [Table] VALUES ( 3,'C',728 )
INSERT [Table] VALUES (11,'C',1050 )
INSERT [Table] VALUES ( 9,'C',3191 )
INSERT [Table] VALUES (17,'C',866 )
INSERT [Table] VALUES ( 4,'C',838 )
INSERT [Table] VALUES (10,'D',550 )
INSERT [Table] VALUES (18,'D',1434 )
IF object_id('tempdb..#tempTable') IS NOT NULL
BEGIN
DROP TABLE #tempTable
END
CREATE TABLE #tempTable
(RowID int, RowCode char(1), RowValue int,RowChunk int)
INSERT INTO #tempTable
select RowID,RowCode,RowValue,null from [table]
declare #RowId int
declare #RowCode char(1)
declare #Count int
declare #CurrentCode char(1)
declare #CountCurrent int
set #Count=1
set #CurrentCode=1
set #CountCurrent=0
DECLARE contact_cursor CURSOR FOR
SELECT RowID,RowCode FROM [table]
OPEN contact_cursor
FETCH NEXT FROM contact_cursor into #RowId,#RowCode
set #CurrentCode=#RowCode
WHILE ##FETCH_STATUS = 0
BEGIN
if(#CurrentCode=#RowCode)
begin
if(#CountCurrent=5)
begin
set #CountCurrent=1
set #Count=#Count+1
update #tempTable set RowChunk=#Count where RowID=#RowID
end
else
begin
set #CountCurrent=#CountCurrent+1
update #tempTable set RowChunk=#Count where RowID=#RowID
end
end
else
begin
set #CurrentCode=#RowCode
set #CountCurrent=1
set #Count=#Count+1
update #tempTable set RowChunk=#Count where RowID=#RowID
end
FETCH NEXT FROM contact_cursor into #RowId,#RowCode
END
CLOSE contact_cursor
DEALLOCATE contact_cursor
select * from #tempTable
GO