Comma separated string total count by variable by row - sql-server

This is the schema:
User_ID Page_ID Timestamp
1 48,51,94 7/26/2017 8:30
2 42,11,84 7/26/2017 9:40
3 4,16,24 7/26/2017 16:20
4 7,2,94 7/27/2017 8:00
1 48,22,94 7/27/2017 13:50
2 42,11 7/27/2017 14:00
3 4,24 7/27/2017 18:15
The code below gives aggregate count of page ids ran per user (non-unique on purpose):
SELECT User_ID, sum(len(Page_ID) - len(replace(Page_ID, ',', '')) +1) as TotalPageCount
FROM DBTABLE
group by User_ID
Output:
User_ID TotalPageCount
1 6
2 5
3 5
4 3
However, I am looking to add a (comma separated) column with page count per page id per user id. ie. a column as newsletter id 1: count, newsletter id 2: count, etc. (essentially a dictionary). Can be a different format, but needs to be descriptive at the page id level, with its respective count.
Something like this:
User_ID PageIDCount TotalPageCount
1 48:2, 51:1, 94:2, 22:1, 6
2 42:2, 11:2, 84:1, 5
3 4:2, 16:1, 24:2, 5
4 7:1, 2:1, 94:1, 3
Your help is greatly appreciated!
Edit:
As per SeanLange's amazing solution, you can change the definition to MyCTE to the below, in order to avoid using any functions:
select user_id, page_id, page_count = count(*)
FROM (
SELECT user_id, Split.a.value('.', 'NVARCHAR(max)') AS page_id FROM
( SELECT user_id, CAST ('<M>' + REPLACE(page_id, ',', '</M><M>') + '</M>' AS XML) page_id
FROM #temp
) AS A
CROSS APPLY page_id.nodes ('/M') AS Split(a)
) x
group by user_id, page_id

Wow this is a nightmare. You are going to need a string splitter to start with. My personal favorite is this one. http://www.sqlservercentral.com/articles/Tally+Table/72993/ There are a number of other excellent choices here. https://sqlperformance.com/2012/07/t-sql-queries/split-strings
Starting with your data you will need to do something like this.
declare #Something table
(
User_ID int
, Page_ID varchar(100)
, MyDate datetime
)
insert #Something
select 1, '48,51,94', '7/26/2017 8:30' union all
select 2, '42,11,84', '7/26/2017 9:40' union all
select 3, '4,16,24', '7/26/2017 16:20' union all
select 4, '7,2,94', '7/27/2017 8:00' union all
select 1, '48,22,94', '7/27/2017 13:50' union all
select 2, '42,11', '7/27/2017 14:00' union all
select 3, '4,24', '7/27/2017 18:15'
select User_ID
, Page_ID = x.Item
, count(*)
from #Something s
cross apply dbo.DelimitedSplit8K(s.Page_ID, ',') x
group by User_ID
, x.Item
order by User_ID
, x.Item
This gets the data with the counts you want. From there you are going to have to shove this back into the denormalized structure that you want. You can do this with FOR XML. Here is an article that explains how to do that part of this. Simulating group_concat MySQL function in Microsoft SQL Server 2005?
-----EDIT-----
OK here is the complete working solution. You have obviously been working hard at trying to get this sorted out. I am using the DelimitedSplit8K function here so I didn't have to inline XML like your solution was doing.
with MyCTE as
(
select User_ID
, Page_ID = x.Item
, PageCount = count(*)
from #Something s
cross apply dbo.DelimitedSplit8K(s.Page_ID, ',') x
group by User_ID
, x.Item
)
, GroupedPageViews as
(
select c.User_ID
, sum(c.PageCount) as TotalPageCount
, PageViews = STUFF((select ', ' + convert(varchar(4), c2.Page_ID) + ':' + convert(varchar(4), c2.PageCount)
from MyCTE c2
where c.User_ID = c2.User_ID
order by c2.Page_ID
for xml path('')), 1, 1, '')
from MyCTE c
group by c.User_ID
)
select gpv.User_ID
, gpv.PageViews
, gpv.TotalPageCount
from GroupedPageViews gpv
join MyCTE c on c.User_ID = gpv.User_ID
group by gpv.PageViews
, gpv.User_ID
, gpv.TotalPageCount
order by gpv.User_ID
This will return your data like this.
User_ID PageViews TotalPageCount
1 22:1, 48:2, 51:1, 94:2 6
2 11:2, 42:2, 84:1 5
3 16:1, 24:2, 4:2 5
4 2:1, 7:1, 94:1 3

Here you go
SELECT DISTINCT User_Id
, (
SELECT CAST(t.Value AS VARCHAR) + ':' + CAST(COUNT(t.value) AS VARCHAR) + ', '
FROM TBL_46160346_DBTABLE ii
CROSS APPLY (
SELECT *
FROM fn_ParseText2Table(Page_ID, ',')
) t
WHERE pp.User_Id = ii.User_Id
GROUP BY User_Id
, VALUE
ORDER BY User_Id
FOR XML PATH('')
) PageIDCount
, (
SELECT COUNT(*)
FROM TBL_46160346_DBTABLE ii
CROSS APPLY (
SELECT *
FROM fn_ParseText2Table(Page_ID, ',')
) t
WHERE pp.User_Id = ii.User_Id
GROUP BY User_Id
) TotalPageCount
FROM TBL_46160346_DBTABLE pp
fn_ParseText2Table function
ALTER FUNCTION [dbo].[fn_ParseText2Table] (
#p_SourceText VARCHAR(8000), #p_Delimeter VARCHAR(10) = ',' --default comma
)
RETURNS #retTable TABLE (Value BIGINT)
AS
BEGIN
DECLARE #w_Continue INT, #w_StartPos INT, #w_Length INT, #w_Delimeter_pos INT, #w_tmp_txt VARCHAR(48), #w_Delimeter_Len TINYINT
IF LEN(#p_SourceText) = 0
BEGIN
SET #w_Continue = 0 -- force early exit
END
ELSE
BEGIN
-- parse the original #p_SourceText array into a temp table
SET #w_Continue = 1
SET #w_StartPos = 1
SET #p_SourceText = RTRIM(LTRIM(#p_SourceText))
SET #w_Length = DATALENGTH(RTRIM(LTRIM(#p_SourceText)))
SET #w_Delimeter_Len = LEN(#p_Delimeter)
END
WHILE #w_Continue = 1
BEGIN
SET #w_Delimeter_pos = CHARINDEX(#p_Delimeter, SUBSTRING(#p_SourceText, #w_StartPos, #w_Length - #w_StartPos + #w_Delimeter_Len))
IF #w_Delimeter_pos > 0 -- delimeter(s) found, get the value
BEGIN
SET #w_tmp_txt = LTRIM(RTRIM(SUBSTRING(#p_SourceText, #w_StartPos, #w_Delimeter_pos - 1)))
SET #w_StartPos = #w_Delimeter_pos + #w_StartPos + #w_Delimeter_Len - 1
END
ELSE -- No more delimeters, get last value
BEGIN
SET #w_tmp_txt = LTRIM(RTRIM(SUBSTRING(#p_SourceText, #w_StartPos, #w_Length - #w_StartPos + #w_Delimeter_Len)))
SELECT #w_Continue = 0
END
INSERT INTO #retTable
VALUES (#w_tmp_txt)
END
RETURN
END

Related

Split a string in SQL by hyphen in 2012 version

I have multiple string in a column where I have get last string after column
Below are three example like same I have different number hyphen that can occur in a string but desired result is I have string before last hyphen
1. abc-def-Opto
2. abc-def-ijk-5C-hello-Opto
3. abc-def-ijk-4C-hi-Build
4. abc-def-ijk-4C-123-suppymanagement
Desired result set is
def
hello
hi
123
How to do this in SQL query to get this result set. I have MSSQL 2012 version
Require a generic sql which can get the result set
There are many ways to split/parse a string. ParseName() would fail because you may have more than 4 positions.
One option (just for fun), is to use a little XML.
We reverse the string
Convert into XML
Grab the second node
Reverse the desired value for the final presentation
Example
Declare #YourTable Table ([SomeCol] varchar(50))
Insert Into #YourTable Values
('abc-def-Opto')
,('abc-def-ijk-5C-hello-Opto')
,('abc-def-ijk-4C-hi-Build')
,('abc-def-ijk-4C-123-suppymanagement')
Select *
,Value = reverse(convert(xml,'<x>'+replace(reverse(SomeCol),'-','</x><x>')+'</x>').value('x[2]','varchar(150)'))
from #YourTable
Returns
SomeCol Value
abc-def-Opto def
abc-def-ijk-5C-hello-Opto hello
abc-def-ijk-4C-hi-Build hi
abc-def-ijk-4C-123-suppymanagement 123
Without getting into XML stuff, simply using string functions of sql server.
Declare #YourTable Table ([SomeCol] varchar(50))
Insert Into #YourTable Values
('abc-def-Opto')
,('abc-def-ijk-5C-hello-Opto')
,('abc-def-ijk-4C-hi-Build')
,('abc-def-ijk-4C-123-suppymanagement');
SELECT *
,RTRIM(LTRIM(REVERSE(
SUBSTRING(
SUBSTRING(REVERSE([SomeCol]) , CHARINDEX('-', REVERSE([SomeCol])) +1 , LEN([SomeCol]) )
, 1 , CHARINDEX('-', SUBSTRING(REVERSE([SomeCol]) , CHARINDEX('-', REVERSE([SomeCol])) +1 , LEN([SomeCol]) ) ) -1
)
)))
FROM #YourTable
i am not sure this script will exactly useful to your requirement but i am just trying to give an idea how to split the data
IF OBJECT_ID('tempdb..#Temp')IS NOT NULL
DROP TABLE #Temp
;WITH CTE(Id,data)
AS
(
SELECT 1,'abc-def-Opto' UNION ALL
SELECT 2,'abc-def-ijk-5C-hello-Opto' UNION ALL
SELECT 3,'abc-def-ijk-4C-hi-Build' UNION ALL
SELECT 4,'abc-def-ijk-4C-123-suppymanagement'
)
,Cte2
AS
(
SELECT Id, CASE WHEN Id=1 AND Setdata=1 THEN data
WHEN Id=2 AND Setdata=2 THEN data
WHEN Id=3 AND Setdata=3 THEN data
WHEN Id=4 AND Setdata=4 THEN data
ELSE NULL
END AS Data
FROM
(
SELECT Id,
Split.a.value('.','nvarchar(1000)') AS Data,
ROW_NUMBER()OVER(PARTITION BY id ORDER BY id) AS Setdata
FROM(
SELECT Id,
CAST('<S>'+REPLACE(data ,'-','</S><S>')+'</S>' AS XML) AS data
FROM CTE
) AS A
CROSS APPLY data.nodes('S') AS Split(a)
)dt
)
SELECT * INTO #Temp FROM Cte2
SELECT STUFF((SELECT DISTINCT ', '+ 'Set_'+CAST(Id AS VARCHAR(10))+':'+Data
FROM #Temp WHERE ISNULL(Data,'')<>'' FOR XML PATH ('')),1,1,'')
Result
Set_1:abc, Set_2:def, Set_3:ijk, Set_4:4C
You can do like
WITH CTE AS
(
SELECT 1 ID,'abc-def-Opto' Str
UNION
SELECT 2, 'abc-def-ijk-5C-hello-Opto'
UNION
SELECT 3, 'abc-def-ijk-4C-hi-Build'
UNION
SELECT 4, 'abc-def-ijk-4C-123-suppymanagement'
)
SELECT ID,
REVERSE(LEFT(REPLACE(P2, P1, ''), CHARINDEX('-', REPLACE(P2, P1, ''))-1)) Result
FROM (
SELECT LEFT(REVERSE(Str), CHARINDEX('-', REVERSE(Str))) P1,
REVERSE(Str) P2,
ID
FROM CTE
) T;
Returns:
+----+--------+
| ID | Result |
+----+--------+
| 1 | def |
| 2 | hello |
| 3 | hi |
| 4 | 123 |
+----+--------+
Demo

TSQL, change value on a comma delimited column

I have a column called empl_type_multi which is just a comma delimited column, each value is a link to another table called custom captions.
For instance, i might have the following as a value in empl_type_multi:
123, RHN, 458
Then in the custom_captions table these would be individual values:
123 = Dog
RHN = Cat
458 = Rabbit
All of these fields are NTEXT.
What i am trying to do is convert the empl_type_multi column and chance it to the respective names in the custom_captions table, so in the example above:
123, RHN, 458
Would become
Dog, Cat, Rabbit
Any help on this would be much appreciated.
----- EDIT ------------------------------------------------------------------
Ok so ive managed to convert the values to the corresponding caption and put it all into a temporary table, the following is the output from a CTE query on the table:
ID1 ID2 fName lName Caption_name Row_Number
10007 22841 fname1 lname1 DENTAL ASSISTANT 1
10007 22841 fname1 lname1 2
10007 22841 fname1 lname1 3
10008 23079 fname2 lname2 OPS WARD 1
10008 23079 fname2 lname2 DENTAL 2
10008 23079 fname2 lname2 3
How can i update this so that anything under caption name is added to the caption name of Row_Number 1 separated by a comma?
If i can do that all i need to do is delete all records where Row_Number != 1.
------ EDIT --------------------------------------------------
The solution to the first edit was:
WITH CTE AS
(
SELECT
p.ID1
, p.ID2
, p.fname
, p.lname
, p.caption_name--
, ROW_NUMBER() OVER (PARTITION BY p.id1ORDER BY caption_name DESC) AS RN
FROM tmp_cs p
)
UPDATE tblPerson SET empType = empType + ', ' + c.Data
FROM CTE c WHERE [DB1].dbo.tblPerson.personID = c.personID AND RN = 2
And then i just incremented RN = 2 until i got 0 rows affected.
This was after i ran:
DELETE FROM CTE WHERE RN != 1 AND Caption_name = ''
select ID1, ID2, fname, lname, left(captions, len(captions) - 1) as captions
from (
select distinct ID1, ID2, cast(fname as nvarchar) as fname, cast(lname as nvarchar) as lname, (
select cast(t1.caption_name as nvarchar) + ','
from #temp as t1
where t1.ID1 = t2.ID1
and t1.ID2 = t2.ID2
and cast(caption_name as nvarchar) != ''
order by t1.[row_number]
for xml path ('')) captions
from #temp as t2
) yay_concatenated_rows
This will give you what you want. You'll see casting from ntext to varchar. This is necessary for comparison because many logical ops can't be performed on ntext. It can be implicitly cast back the other way so no worries there. Note that when casting I did not specify length; this will default to 30, so adjust as varchar(length) as needed to avoid truncation. I also assumed that both ID1 and ID2 form a composite key (it appears this is so). Adjust the join as you need for the relationship.
you have just shared your part of problem,not exact problem.
try this,
DECLARE #T TABLE(ID1 VARCHAR(50),ID2 VARCHAR(50),fName VARCHAR(50),LName VARCHAR(50),Caption_name VARCHAR(50),Row_Number INT)
INSERT INTO #T VALUES
(10007,22841,'fname1','lname1','DENTAL ASSISTANT', 1)
,(10007,22841,'fname1','lname1', NULL, 2)
,(10007,22841,'fname1','lname1', NULL, 3)
,(10008,23079,'fname2','lname2','OPS WARD', 1)
,(10008,23079,'fname2','lname2','DENTAL', 2)
,(10008,23079,'fname2','lname2', NULL, 3)
SELECT *
,STUFF((SELECT ','+Caption_name
FROM #T T1 WHERE T.ID1=T1.ID1 FOR XML PATH('')
),1,1,'')
FROM #T T
You can construct the caption_name string easily by looping through while loop
declare #i int = 2,#Caption_name varchar(100)= (select series from
#temp where Row_Number= 1)
while #i <= (select count(*) from #temp)
begin
select #Caption_name = #Caption_name + Caption_name from #temp where Row_Number = #i)
set #i = #i+1
end
update #temp set Caption_name = #Caption_name where Row_Number = 1
and use case statement to remove null values
(select case when isnull(Caption_name ,'') = '' then
'' else ',' + Caption_name end

SQL Server array_agg (master - detail)

How can I convert this PostgreSQL code to SQL Server ?
select
countries.title,
(select array_to_json(array_agg(row_to_json(t)))
from postcodes t
where t.country_id = countries.id) as codes
from countries
My initial problem is that I need to select complete master table and with each row all details.
Countries:
id title
1 SLO
2 AUT
PostCodes:
id country_id code title
1 1 1000 Lj
2 1 2000 Mb
3 2 22180 Vi
4 2 22484 De
Desired result:
1 SLO 1000;Lj|2000;MB
2 AUT 22180;Vi|22484;De
Not:
1 SLO 1000 Lj
1 SLO 2000 Mb
2 AUT 22180 Vi
2 AUT 22484 De
The best solution would be using FOR JSON, but unfortunately I need support for 2008 or at least 2012.
With left join all master data are duplicated for detail count, but I do not want to do this. Even worse it would be to select all countries and then call select on post_codes for every country in for loop.
select countries.title,
STUFF((select '|' + t.code + ';' + t.title
from postcodes t
where t.country_id = countries.id
FOR XML PATH('')
),1,1,'') as codes
from countries
-- CAST t.code to VARCHAR if it's Number
try this:
Select Main.COUNTRY_ID,c.title,Left(Main.POSTCODES,Len(Main.POSTCODES)-1) As "POSTCODES"
From
(
Select distinct ST2.COUNTRY_ID,
(
Select ST1.CODE+';'+ST1.TITLE + '|' AS [text()]
From dbo.POSTCODES ST1
Where ST1.COUNTRY_ID = ST2.COUNTRY_ID
ORDER BY ST1.COUNTRY_ID
For XML PATH ('')
) [POSTCODES]
From dbo.POSTCODES ST2
) [Main]
inner join countries c on c.id=main.country_id
Using XML PATH for concatenation can increase the complexity of your code. It's better to implement a CLR aggregation function. Then, you can do the following:
SELECT C.[id]
,C.[title]
,REPLACE([dbo].[Concatenate] (P.[code] + ';' + P.[title]), ',', '|')
FROM #Countries C
INNER JOIN #PostCodes P
ON C.[id] = p.[country_id]
GROUP BY C.[id]
,C.[title];
You can create your own version of the concatenate aggregate - you can specify the delimiter, the order, etc. I can show you examples if you want.
DECLARE #Countries TABLE
(
[id] TINYINT
,[title] VARCHAR(12)
);
INSERT INTO #Countries ([id], [title])
VALUES (1, 'SLO')
,(2, 'AUT');
DECLARE #PostCodes TABLE
(
[id] TINYINT
,[country_id] TINYINT
,[code] VARCHAR(12)
,[title] VARCHAR(12)
);
INSERT INTO #PostCodes ([id], [country_id], [code], [title])
VALUES (1, 1, 1000, 'Lj')
,(2, 1, 2000, 'Mb')
,(3, 2, 22180, 'Vi')
,(4, 2, 22484, 'De');
SELECT C.[id]
,C.[title]
,REPLACE([dbo].[Concatenate] (P.[code] + ';' + P.[title]), ',', '|')
FROM #Countries C
INNER JOIN #PostCodes P
ON C.[id] = p.[country_id]
GROUP BY C.[id]
,C.[title];

TSQL While Loop over months in year

I have an output that I need to achieve and I am not too certain how to go about it.
I first need to start by looping over each month in the year and using that month in a select statement to check for data.
For example:
Select * from table where MONTH(A.[submissionDate]) = 1
Select * from table where MONTH(A.[submissionDate]) = 2
Select * from table where MONTH(A.[submissionDate]) = 3
My end result is to create this XML output to use with a chart plugin. It needs to include the months even if there is no data which is why I wanted to loop through each month to check for it.
<root>
<dataSet>
<areaDesc>Area 1</areaDesc>
<data>
<month>January</month>
<monthValue>1</monthValue>
<submissions>0</submissions>
</data>
<data>
<month>February</month>
<monthValue>2</monthValue>
<submissions>7</submissions>
</data>
<data>
<month>March</month>
<monthValue>3</monthValue>
<submissions>5</submissions>
</data>
</dataSet>
<dataSet>
<areaDesc>Area 2</areaDesc>
<data>
<month>January</month>
<monthValue>1</monthValue>
<submissions>0</submissions>
</data>
<data>
<month>February</month>
<monthValue>2</monthValue>
<submissions>7</submissions>
</data>
<data>
<month>March</month>
<monthValue>3</monthValue>
<submissions>5</submissions>
</data>
</dataSet>
</root>
I may be way over thinking this but I'm hoping I talking it through may help me out a little.
Here is my current set up of how I get some other stats:
--Temp table
DECLARE #areas TABLE (
area VARCHAR (100));
IF #dept = 'global'
OR #dept = ''
BEGIN
INSERT INTO #areas (area)
SELECT DISTINCT(AreaDesc)
FROM dbo.EmpTable;
END
ELSE
BEGIN
INSERT INTO #areas
SELECT #dept;
END
IF (#action = 'compare')
BEGIN
SELECT DATENAME(month, A.[submissionDate]) AS [month],
MONTH(A.[submissionDate]) AS [monthValue],
count(A.[submissionID]) AS submissions,
B.[AreaDesc]
FROM empowermentSubmissions AS A
INNER JOIN empTable AS B
ON A.[nomineeQID] = B.[QID]
WHERE YEAR(A.[submissionDate]) = #year
AND A.[statusID] = 3
AND A.[locationID] IN (SELECT location
FROM #table)
GROUP BY DATENAME(month, A.[submissionDate]), MONTH(A.[submissionDate]), B.[AreaDesc]
ORDER BY [monthValue] ASC
FOR XML PATH ('dataSet'), TYPE, ELEMENTS, ROOT ('root');
END
ELSE
This is a great application for a "Dates" table or view. Create a new table in your database with schema like:
CREATE TABLE dbo.Dates (
Month INT,
MonthName VARCHAR(20)
)
Populate this table with the years and months you may want to aggregate over. Then, you can make your query like:
SELECT
Area
Dates.MonthName,
COUNT(*) AS Count
FROM
dbo.Dates
LEFT OUTER JOIN
dbo.Submissions
AND Dates.Month = MONTH(Submissions.SubmissionDate)
GROUP BY
Dates.MonthName,
Area
The LEFT OUTER JOIN will give you one row for every Year and Month in the dates table, and a count of any submissions on that month. You end up with output like:
Area | MonthName | Count
Area 1 | Jan | 0
Area 2 | Feb | 2
&c.
You'll want to do a FOR XML structure to get the exact result set you're looking for in one go, I think. I put this together with what I could glean about your XML. Just change the name of the table variable here to your real table name and this should work.
EDIT: changed up the query to match the definition from the posted query. Updated the data element where clause to maintain month instantiation when zero counts were found in a month.
EDIT: Added Status requirement.
EDIT: Moved areaDesc criteria for constant month output.
declare #empowermentSubmissions table (submissionID int primary key identity(1,1), submissionDate datetime, nomineeQID INT, statusID INT)
declare #empTable table (QID int primary key identity(1,1), AreaDesc varchar(10))
declare #n int = 1
while #n < 50
begin
insert into #empTable (AreaDesc) values ('Area ' + cast((#n % 2)+1 as varchar(1)))
set #n = #n + 1
end
set #n = 1
while #n < 500
begin
insert into #empowermentSubmissions (submissionDate, nomineeQID, StatusID) values (dateadd(dd,-(cast(rand()*600 as int)),getdate()), (select top 1 QID from #empTable order by newid()), 3 + (#n % 2) - (#n % 3) )
set #n = #n + 1
end
declare #year int = 2014
select (
select (
select (
select e1.areaDesc
from #empTable e1
where e1.areaDesc = e2.areaDesc
group by e1.areaDesc
for xml path(''),type
)
, (
select [month], [monthValue], count(s1.submissionID) as submissions
from (
select #year [Year]
, datename(month,dateadd(mm,RowID-1,#year-1900)) [Month]
, month(dateadd(mm,RowID-1,#year-1900)) [MonthValue]
from (
select *, row_number()over(order by name) as RowID
from master..spt_values
) d
where d.RowID <= 12
) t
left join (
select s3.submissionID, s3.submissionDate, e3.AreaDesc
from #empowermentSubmissions s3
inner join #empTable e3 on s3.nomineeQID = e3.QID
where s3.statusID = 3
and e3.areaDesc = e2.areaDesc
) s1 on year(s1.submissionDate) = t.[Year]
and month(s1.submissionDate) = t.[MonthValue]
group by [Month], [MonthValue]
order by [MonthValue]
for xml path('data'),type
)
for xml path(''),type
) dataset
from #empowermentSubmissions s2
inner join #empTable e2 on s2.nomineeQID = e2.QID
group by e2.areaDesc
for xml path(''), type
) root
for xml path (''), type
You should be able to use a tally table to get the months:
SELECT TOP 12 IDENTITY(INT,1,1) AS N
INTO #tally
FROM master.dbo.syscolumns sc1
SELECT DATENAME(MONTH,DATEADD(MONTH,t.N-1,'2014-01-01')) AS namemonth, t.N AS monthvalue, COUNT(tbl.submissionDate) AS submissions, tbl.Area
FROM #tally t
LEFT OUTER JOIN tbl ON MONTH(tbl.submissionDate) = t.N
GROUP BY t.n, tbl.Area
DROP TABLE #tally

Avoid WHILE loop and CURSOR for better performance?

I'm wondering if someone can help simplify this procedure - and improve performance...!?
We have data on grants. 'Donors' give funds to 'Recipients' and we want to show the top 15 recipients for each donor over 3 periods: CurrentYear-20, CurrentYear-10 and CurrentYear. We publish an annual report and show percentage shares of World and GeoZone totals for each donor.
I have "inherited" this code which was written by one of my predecessors. Until we switched to using a view, execution time was around 15-30 mins. Currently, this runs in just under FOUR hours (scheduled as a Server Agent job)! Management are not happy. For various reasons, the view must continue to be used and currently has just under 900,000 rows with data from the 1950s onwards. We current run this report for 30 (large) donors and more are added each year.
To help improve performance, I have thought about using a CTE or/using SUM() OVER(Partition BY...) or combination of these, but I'm not sure how to go about it.
Could someone point me in the right direction?
Here is the process:
create a table (variable) to hold the top 15 recipients for the current donor
create a table (variable) to hold the list of donors
populate the donor table with the donors in the order they appear in the report
loop thru the donor table and for each donor:
put the donor ID for this donor into a temp table
loop 3 times (for CurrentYear-20, CurrentYear-10, CurrentYear)
calculate the share totals for each of 18 regions/zones
print the values for each section in the report
get the next donor ID
As you may see from the above, the calculations are run 54 times (18x3) for each donor!
Here is the code (simplified):
-- #LatestYear is passed as a parameter, hardcoded here for simplicity
DECLARE #LatestYear SMALLINT ,
#CurrentYear SMALLINT ,
#DonorID SMALLINT ,
#totalWorld NUMERIC(10, 2) ,
#LoopCounter TINYINT ,
#DonorName VARCHAR(100)
SELECT #latestyear = 2012
-- create a table to hold list of top 15 recipients for each donor and their 'share' of ODA.
DECLARE #Top15 TABLE
(
Country VARCHAR(100) ,
Percentage REAL
)
-- create a table to hold list of donors, ordered as they need to appear in the report.
DECLARE #PageOrder TABLE
(
DonorID SMALLINT ,
DonorName VARCHAR(100) ,
SortOrder SMALLINT IDENTITY(1, 1)
)
-- create a table to store the "focus" donor.
DECLARE #CurrentDonor TABLE ( DonorID SMALLINT )
INSERT INTO #PageOrder
SELECT DonorID ,
DonorName
FROM dbo.LookupDonor
ORDER BY DonorName;
-- cursor to loop through the donors in SortOrder
DECLARE DonorCursor CURSOR
FOR
SELECT DonorID ,
DonorName
FROM #PageOrder
ORDER BY DonorName;
OPEN DonorCursor
FETCH NEXT FROM DonorCursor INTO #DonorID, #DonorName
WHILE ##fetch_status = 0
BEGIN
INSERT INTO pubOutput
( XMLText )
SELECT #DonorName;
-- Populate the DonorID table
INSERT INTO #CurrentDonor
VALUES ( #DonorID )
/* The following loop is invoked 3 times. The first time through, the year will be 20 years before the latest year,
the second time through, 10 years before. The last time through the year will be the latest year.
*/
SET #LoopCounter = 1
WHILE #LoopCounter <= 3
BEGIN
SELECT #CurrentYear = CASE #LoopCounter
WHEN 1 THEN #LatestYear - 20
WHEN 2 THEN #LatestYear - 10
ELSE #LatestYear
END
-- calculate the world total for the current years (year,year-1) for all recipients
SELECT #totalWorld = SUM(Amount)
FROM dbo.vData2 d
INNER JOIN ( SELECT RecipientID
FROM dbo.RecipientGroup
WHERE GroupID = 160
) c ON d.RecipientID = c.RecipientID
INNER JOIN #CurrentDonor z ON d.DonorID = z.DonorID
WHERE d.year IN ( #CurrentYear - 1, #CurrentYear )
-- calculate the GeoZones total for the current years (year,year-1)
SELECT #totalGeoZones = SUM(Amount)
FROM dbo.vDac2a d
INNER JOIN ( SELECT RecipientID
FROM dbo.GeoZones
WHERE GeoZoneID = 100
) x ON d.RecipientID = x.RecipientID
INNER JOIN #CurrentDonor z ON d.DonorCode = z.DonorCode
WHERE d.year IN ( #CurrentYear - 1, #CurrentYear )
-- Find the top 15 recipients for the current donor
INSERT INTO #Top15
SELECT TOP 15
r.RecipientName ,
( ISNULL(SUM(Amount), 0) / #totalWorld ) * 100
FROM dbo.vData2 d
INNER JOIN dbo.LookupRecipient r ON r.RecipientID = d.RecipientID
INNER JOIN #CurrentDonor z ON d.DonorID = z.DonorID
WHERE d.year IN ( #CurrentYear - 1, #CurrentYear )
GROUP BY r.RecipientName
ORDER BY 2 DESC
-- Print the top 15 recipients and total
INSERT INTO pubOutput
(
XMLText
)
SELECT country + #Separator + CAST(percentage AS VARCHAR)
FROM #Top15
ORDER BY percentage DESC
INSERT INTO pubOutput
(
XMLText
)
SELECT #Heading1 + #Separator + CAST(SUM(Percentage) AS VARCHAR)
FROM #Top15
-- Breakdown by Regionas
-- Region1
IF #totalWorld IS NOT NULL
INSERT INTO pubOutput
(
XMLText
)
SELECT 'Region1' + #Separator
+ CAST(( ISNULL(SUM(Amount), 0) / #totalWorld ) * 100 AS VARCHAR)
FROM dbo.vData2 d
INNER JOIN ( SELECT RecipientID
FROM dbo.RecipientGroup
WHERE RegionID = 1
) c ON d.RecipientID = c.RecipientID
INNER JOIN #CurrentDonor z ON d.DonorID = z.DonorID
WHERE d.year IN ( #CurrentYear - 1, #CurrentYear )
ELSE -- force output of sub-total heading
INSERT INTO pubOutput
(
XMLText
)
SELECT #Heading2 + #Separator + '--'
-- Region2-8
/* similar syntax as Region1 above, for all Regions 2-8 */
-- Total Regions
INSERT INTO pubOutput
(
XMLText
)
SELECT #Heading2 + #Separator + CAST(#totalWorld AS VARCHAR)
-- Breakdown by GeoZones 1-7
-- GeoZone1
INSERT INTO pubOutput
(
XMLText
)
SELECT 'GeoZone1' + #Separator
+ CAST(( ISNULL(SUM(Amount), 0) / #totalGeoZones ) * 100 AS VARCHAR)
FROM dbo.vDac2a d
INNER JOIN ( SELECT RecipientID
FROM dbo.GeoZones
WHERE GeoZoneID = 1
) m ON d.RecipientID = m.RecipientID
INNER JOIN #CurrentDonor z ON d.DonorCode = z.DonorCode
WHERE d.year IN ( #CurrentYear - 1, #CurrentYear )
-- GeoZones2-8
/* similar syntax as GeoZone1 above for GeoZones 2-7 */
-- Total GeoZones - currently hard-coded as 100, due to minor rounding errors
INSERT INTO pubOutput
(
XMLText
)
SELECT #Heading3 + #Separator + '100'
SET #LoopCounter = #LoopCounter + 1
END -- year loop
-- Get the next donor from the cursor
FETCH NEXT FROM DonorCursor
INTO #DonorID, #DonorName
END
-- donorcursor
-- Cleanup
CLOSE DonorCursor
DEALLOCATE DonorCursor
Many thanks in advance for any help you may be able to provide.
Avoiding cursor is must. You can use 'while' instead of cursor. However considering the complexity of query, keep cursor at this moment.
To improve performance in other way, check the number of records for below queries:
SELECT RecipientCode FROM dbo.RecipientGroup WHERE GroupID=160
SELECT RecipientCode FROM dbo.GeoZones WHERE GeoZoneID=100
SELECT RecipientID FROM dbo.RecipientGroup WHERE RegionID=1
I suggest create 3 temp tables for above query "outside" of cursor and use them inside of cursor.
Hope this helps!

Resources