How to suppress displaying specific columns on dynamic tables - sql-server

I have two dynamic tables that I am left joining. I want to suppress displaying specific column from both tables. So far, this is my query:
SELECT a.*
,b.*
FROM APEXCOUNTYREVIEW_REPORT a
LEFT JOIN TBL_LOW_VOLUME_test b
ON a.fips_data_type = b.fips_datatype
Both of these are dynamic tables with plenty of columns. I want to suppress columns in the middle of the tables.
For example, Table1 has columns C1, C2, C3, C4, C5, C6, C7,...etc. I do not want to pull columns C3, C4 and C5, but want to display C1, C2, C6, C7 and so forth. Since both of these are dynamic tables, the number of columns is not known until execution of the query. I'm using SSMS 2008 R2.
Is there a way to do this with SQL query?
Here's the query to build table2, tbl_low_voulume_test:
I'm also trying to add a text to the column name on table2, since they are the same as the column names on table1, that is being created dynamically and I'm getting all kinds of invalid error messages. How can I add the text "Flag" on the dynamic column names?
The query below runs fine.
DECLARE #T AS TABLE(Y INT NOT NULL PRIMARY KEY);
DECLARE
#COLS AS NVARCHAR(MAX),
#COLNAMES AS NVARCHAR(MAX),
#Y AS INT,
#SQL AS NVARCHAR(MAX);
SET #COLS = STUFF(
(SELECT N',' + QUOTENAME(Y)
FROM (SELECT DISTINCT RECMONTH AS Y FROM REPORTING.dbo.TBL_LOW_VOLUME A
WHERE (RECMONTH>= '2005-01')
AND RECMONTH <= CONVERT(VARCHAR(7), getdate(), 120)
) AS Y
ORDER BY Y
FOR XML PATH('')),
1, 1, N'');
SET #COLNAMES = STUFF(
(SELECT N',ISNULL(' + QUOTENAME(Y)+',0) AS '+ QUOTENAME(Y)
FROM (SELECT DISTINCT RECMONTH AS [Y] FROM REPORTING.dbo.TBL_LOW_VOLUME A
WHERE (RECMONTH>= '2005-01')
AND RECMONTH <= CONVERT(VARCHAR(7), getdate(), 120)
) AS Y
ORDER BY Y
FOR XML PATH('')),
1, 1, N'');
IF OBJECT_ID('TBL_LOW_VOLUME_test') IS NOT NULL
BEGIN DROP TABLE reporting.dbo.TBL_LOW_VOLUME_test END
SET #SQL = N'SELECT * INTO dbo.TBL_LOW_VOLUME_test
FROM (
SELECT CONVERT(DATE, GETDATE()) AS RUN_DATE, FIPS+DATATYPE AS FIPS_DATATYPE,
FIPS,DATATYPE,' +#COLNAMES + '
FROM (SELECT A.[FIPS], A.DATATYPE
,RECMONTH
,[LV_FLAG]
FROM dbo.TBL_LOW_VOLUME A
where FIPS is not NULL and isnumeric(FIPS) = 1 and len(ltrim(FIPS)) = 5
and FIPS not in (''-0000'',''00000'',''-0001'')
and FIPS <= ''56999''
) AS D
PIVOT(SUM([LV_FLAG]) FOR RECMONTH IN(' + #COLS + N')) AS P) AS Z';
EXEC SP_EXECUTESQL #SQL;
Here's a picture of my two tables.
Table1:
col1, col2, col3, col4, col5, col6, etc...
columns 1 thru 5 are static, starting on column c0l6, it becomes dynamic where every beginning of the month it adds a new month.
Table2:
cola, colb, colc, cold, cole, colf, etc...
columns a and b are both static, the rest are dynamic where every beginning of the month it adds a new month.
This should be the final results I want to display.
col1, col2, col6, etc...(all the dynamic columns on table1), cola, colc, cole, etc...(all the dynamic columns on table2)
So I want to pick and choose which columns I need from each table.
Here's the query after I did print #sql:
For this table, I do not want to display Run_Date and FIPS_DATATYPE, just all the date columns.
SELECT * INTO dbo.TBL_LOW_VOLUME_test
FROM (
SELECT CONVERT(DATE, GETDATE()) AS RUN_DATE, FIPS+DATATYPE AS FIPS_DATATYPE,
ISNULL([2005-01],0) AS [2005-01],ISNULL([2005-02],0) AS [2005-02],ISNULL([2005-03],0) AS [2005-03],ISNULL([2005-04],0) AS [2005-04],
ISNULL([2005-05],0) AS [2005-05],ISNULL([2005-06],0) AS [2005-06],ISNULL([2005-07],0) AS [2005-07],ISNULL([2005-08],0) AS [2005-08],
ISNULL([2005-09],0) AS [2005-09],ISNULL([2005-10],0) AS [2005-10],ISNULL([2005-11],0) AS [2005-11],ISNULL([2005-12],0) AS [2005-12],
ISNULL([2006-01],0) AS [2006-01],ISNULL([2006-02],0) AS [2006-02],ISNULL([2006-03],0) AS [2006-03],ISNULL([2006-04],0) AS [2006-04],
ISNULL([2006-05],0) AS [2006-05],ISNULL([2006-06],0) AS [2006-06],ISNULL([2006-07],0) AS [2006-07],ISNULL([2006-08],0) AS [2006-08],
ISNULL([2006-09],0) AS [2006-09],ISNULL([2006-10],0) AS [2006-10],ISNULL([2006-11],0) AS [2006-11],ISNULL([2006-12],0) AS [2006-12],
ISNULL([2007-01],0) AS [2007-01],ISNULL([2007-02],0) AS [2007-02],ISNULL([2007-03],0) AS [2007-03],ISNULL([2007-04],0) AS [2007-04],
ISNULL([2007-05],0) AS [2007-05],ISNULL([2007-06],0) AS [2007-06],ISNULL([2007-07],0) AS [2007-07],ISNULL([2007-08],0) AS [2007-08],
ISNULL([2007-09],0) AS [2007-09],ISNULL([2007-10],0) AS [2007-10],ISNULL([2007-11],0) AS [2007-11],ISNULL([2007-12],0) AS [2007-12],
ISNULL([2008-01],0) AS [2008-01],ISNULL([2008-02],0) AS [2008-02],ISNULL([2008-03],0) AS [2008-03],ISNULL([2008-04],0) AS [2008-04],
ISNULL([2008-05],0) AS [2008-05],ISNULL([2008-06],0) AS [2008-06],ISNULL([2008-07],0) AS [2008-07],ISNULL([2008-08],0) AS [2008-08],
ISNULL([2008-09],0) AS [2008-09],ISNULL([2008-10],0) AS [2008-10],ISNULL([2008-11],0) AS [2008-11],ISNULL([2008-12],0) AS [2008-12],
ISNULL([2009-01],0) AS [2009-01],ISNULL([2009-02],0) AS [2009-02],ISNULL([2009-03],0) AS [2009-03],ISNULL([2009-04],0) AS [2009-04],
ISNULL([2009-05],0) AS [2009-05],ISNULL([2009-06],0) AS [2009-06],ISNULL([2009-07],0) AS [2009-07],ISNULL([2009-08],0) AS [2009-08],
ISNULL([2009-09],0) AS [2009-09],ISNULL([2009-10],0) AS [2009-10],ISNULL([2009-11],0) AS [2009-11],ISNULL([2009-12],0) AS [2009-12],
ISNULL([2010-01],0) AS [2010-01],ISNULL([2010-02],0) AS [2010-02],ISNULL([2010-03],0) AS [2010-03],ISNULL([2010-04],0) AS [2010-04],
ISNULL([2010-05],0) AS [2010-05],ISNULL([2010-06],0) AS [2010-06],ISNULL([2010-07],0) AS [2010-07],ISNULL([2010-08],0) AS [2010-08],
ISNULL([2010-09],0) AS [2010-09],ISNULL([2010-10],0) AS [2010-10],ISNULL([2010-11],0) AS [2010-11],ISNULL([2010-12],0) AS [2010-12],
ISNULL([2011-01],0) AS [2011-01],ISNULL([2011-02],0) AS [2011-02],ISNULL([2011-03],0) AS [2011-03],ISNULL([2011-04],0) AS [2011-04],
ISNULL([2011-05],0) AS [2011-05],ISNULL([2011-06],0) AS [2011-06],ISNULL([2011-07],0) AS [2011-07],ISNULL([2011-08],0) AS [2011-08],
ISNULL([2011-09],0) AS [2011-09],ISNULL([2011-10],0) AS [2011-10],ISNULL([2011-11],0) AS [2011-11],ISNULL([2011-12],0) AS [2011-12],
ISNULL([2012-01],0) AS [2012-01],ISNULL([2012-02],0) AS [2012-02],ISNULL([2012-03],0) AS [2012-03],ISNULL([2012-04],0) AS [2012-04],
ISNULL([2012-05],0) AS [2012-05],ISNULL([2012-06],0) AS [2012-06],ISNULL([2012-07],0) AS [2012-07],ISNULL([2012-08],0) AS [2012-08],
ISNULL([2012-09],0) AS [2012-09],ISNULL([2012-10],0) AS [2012-10],ISNULL([2012-11],0) AS [2012-11],ISNULL([2012-12],0) AS [2012-12],
ISNULL([2013-01],0) AS [2013-01],ISNULL([2013-02],0) AS [2013-02],ISNULL([2013-03],0) AS [2013-03],ISNULL([2013-04],0) AS [2013-04],
ISNULL([2013-05],0) AS [2013-05],ISNULL([2013-06],0) AS [2013-06],ISNULL([2013-07],0) AS [2013-07],ISNULL([2013-08],0) AS [2013-08],
ISNULL([2013-09],0) AS [2013-09],ISNULL([2013-10],0) AS [2013-10],ISNULL([2013-11],0) AS [2013-11],ISNULL([2013-12],0) AS [2013-12],
ISNULL([2014-01],0) AS [2014-01],ISNULL([2014-02],0) AS [2014-02],ISNULL([2014-03],0) AS [2014-03],ISNULL([2014-04],0) AS [2014-04],
ISNULL([2014-05],0) AS [2014-05],ISNULL([2014-06],0) AS [2014-06],ISNULL([2014-07],0) AS [2014-07],ISNULL([2014-08],0) AS [2014-08],
ISNULL([2014-09],0) AS [2014-09],ISNULL([2014-10],0) AS [2014-10],ISNULL([2014-11],0) AS [2014-11],ISNULL([2014-12],0) AS [2014-12],
ISNULL([2015-01],0) AS [2015-01],ISNULL([2015-02],0) AS [2015-02]

How can I add the text "Flag" on the dynamic column names?
Looks to me like you want to replace
QUOTENAME(Y)
with
QUOTENAME(Y+'Flag')
Everywhere it occurs in both the SET #COLS and SET #COLNAMES statements.
EDIT: To filter out columns, looking at your final select, you're doing SELECT * FROM (a derived table that specifies static columns + COLNAMES).
So to filter out columns, filter them out of the derived table. They have to be either the static columns RUN_DATE, FIPS_DATATYPE, FIPS, DATATYPE, or one of the columns in COLNAMES, which you can filter out with a REPLACE function.
Or if I am not understanding the problem, please post sample output of your current query and which rows you want to filter out.
2ND EDIT: To take Run_Date and FIPS_DATATYPE out of your results, simply take them out of the middle-level select:
SET #SQL = N'SELECT * INTO dbo.TBL_LOW_VOLUME_test
FROM (
SELECT ' +#COLNAMES + '
FROM ...

Related

SQL - String Manipulation

Context:
I have a view in SQL Server that tracks parameters a user inputs when they run an SSRS report (ReportServer.dbo.ExecutionLog). About 50 report parameters are saved as a string in a single column with ntext datatype. I would like to break this single column up into multiple columns for each parameter.
Details:
I query the report parameters like this:
SELECT ReportID, [Parameters]
FROM ReportServer.dbo.ExecutionLog
WHERE ReportID in (N'redacted')
and [Status] in (N'rsSuccess')
ORDER BY TimeEnd DESC
And here's a small subset of what the results look like:
alpha=123&bravo=9%2C33%2C76%2C23&charlie=91&delta=29&echo=11%2F2%2F2018%2012%3A00%3A00%20AM&foxtrot=11%2F1%2F2030%2012%3A00%3A00%20AM
Quesitons:
How can I get the results to look like this:
SQL Server 2017 is Python friendly. Is Python a better language to use in this scenario just for parsing purposes?
I've seen similar topics posted here, here & here. The parameters are dynamic so parsing via SQL string functions that involve counting characters doesn't apply. This question is relevant to more people than just me because there's a large population of people using SSRS. Tracking & formatting parameters in a more digestible way is valuable for all users of SSRS.
Here is a way using the built in STRING_SPLIT. I'm just not sure what the logic is for the stuff AFTER the date, so I would discarded it but I left it for you to decide.
DEMO
declare #table table (ReportID int identity(1,1), [Parameters] varchar(8000))
insert into #table
values
('alpha=123&bravo=9%2C33%2C76%2C23&charlie=91&delta=29&echo=11%2F2%2F2018%2012%3A00%3A00%20AM&foxtrot=11%2F1%2F2030%2012%3A00%3A00%20AM')
,('alpha=457893&bravo=9%2C33%2C76%2C23&charlie=91&delta=29&echo=11%2F2%2F2018%2012%3A00%3A00%20AM&foxtrot=11%2F1%2F2030%2012%3A00%3A00%20AM')
select
ReportID
,[Parameters]
,alpha = max(iif(value like 'alpha%',substring(value,charindex('=',value) + 1,99),null))
,bravo = max(iif(value like 'bravo%',substring(value,charindex('=',value) + 1,99),null))
,charlie = max(iif(value like 'charlie%',substring(value,charindex('=',value) + 1,99),null))
,delta = max(iif(value like 'delta%',substring(value,charindex('=',value) + 1,99),null))
,echo = max(iif(value like 'echo%',substring(value,charindex('=',value) + 1,99),null))
,foxtrot = max(iif(value like 'foxtrot%',substring(value,charindex('=',value) + 1,99),null))
from #table
cross apply string_split(replace(replace([Parameters],'%2C',','),'%2F','/'),'&')
group by ReportID, [Parameters]
Or, if they aren't static you can use a dynamic pivot. It'll take some massaging to get your columns in the correct order.
DEMO
DECLARE #cols AS NVARCHAR(MAX),
#query AS NVARCHAR(MAX);
SET #cols = STUFF((SELECT distinct ',' + QUOTENAME(substring([value],0,charindex('=',[value])))
from myTable
cross apply string_split(replace(replace([Parameters],'%2C',','),'%2F','/'),'&')
FOR XML PATH(''), TYPE
).value('.', 'NVARCHAR(MAX)')
,1,1,'')
select #cols
set #query = 'SELECT ReportID, ' + #cols + ' from
(
select ReportID
, ColName = substring([value],0,charindex(''='',[value]))
, ColVal = substring([value],charindex(''='',[value]) + 1,99)
from myTable
cross apply string_split(replace(replace([Parameters],''%2C'','',''),''%2F'',''/''),''&'')
) x
pivot
(
max(ColVal)
for ColName in (' + #cols + ')
) p '
execute(#query)
Split the string on the ampersand character.
Further split each row into two columns on the equals character.
In the second column, replace %2C with the comma character, and %2F with the forward-slash character, and so on with any other replacements as needed.
Use a dynamic-pivot to query the above in the format that you want.
Here's a method that starts with a lot of replaces.
To url-decode the string and transform it into an XML type.
Then it uses the XML functions to get the values for the columns.
Example snippet:
declare #Table table ([Parameters] varchar(200));
insert into #Table ([Parameters]) values
('alpha=123&bravo=9%2C33%2C76%2C23&charlie=91&delta=29&echo=11%2F2%2F2018%2012%3A00%3A00%20AM&foxtrot=11%2F1%2F2030%2012%3A00%3A00%20AM');
select
x.query('/x[key="alpha"]/val').value('.', 'int') as alpha,
x.query('/x[key="bravo"]/val').value('.', 'varchar(30)') as bravo,
x.query('/x[key="charlie"]/val').value('.', 'varchar(30)') as charlie,
x.query('/x[key="delta"]/val').value('.', 'varchar(30)') as delta,
convert(date, x.query('/x[key="echo"]/val').value('.', 'varchar(30)'), 103)as echo,
convert(date, x.query('/x[key="foxtrot"]/val').value('.', 'varchar(30)'), 103) as foxtrot
from #Table
cross apply (select cast('<x><key>'+
replace(replace(replace(replace(replace(
replace([Parameters],
'%2C',','),
'%2F','/'),
'%20',' '),
'%3A',':'),
'=','</key><val>'),
'&','</val></x><x><key>')
+'</val></x>' as XML) as x) ca
Test on db<>fiddle here

Exec into temp table without columns as they are dynamic

I'm working on pivot functionality, where the pivot columns comes dynamically. All the pivot columns I needs to count and display separate column in pivot select statement itself.
SELECT CurriculumSegment,attendancedate,section,Class,count(*) as
TotalWorkingDays into #NewTempWithTotalDaysCount
FROM #temp1 group by CurriculumSegment,attendancedate,Section,Class
order by CurriculumSegment, attendancedate,Class,Section
SELECT CurriculumSegment,attendancedate,section,Class,count(*) as
CountOfPresentDays into #NewTempWithTotalPresentDaysCount
FROM #temp1 where attendance='P' group by
CurriculumSegment,attendancedate,Section,Class order by CurriculumSegment,
attendancedate,Class,Section
select a.AttendanceDate,a.Class,a.CurriculumSegment,
a.Section,a.TotalWorkingDays,b.CountOfPresentDays,(a.TotalWorkingDays -
b.CountOfPresentDays) as TotalAbsentDays, (b.CountOfPresentDays *
100)/a.TotalWorkingDays as PercentOfPresentDays
into #TempWithTotalDaysCount
from #NewTempWithTotalDaysCount a left outer join
#NewTempWithTotalPresentDaysCount b
on a.AttendanceDate =b.AttendanceDate and a.CurriculumSegment =
b.CurriculumSegment and a.Section= b.Section and a.Class = b.Class
-- Query 1
select sum(TotalWorkingDays) as TotalCount,sum(TotalAbsentDays) as
AbsentCount,sum(CountOfPresentDays) as
PresentCount,curriculumsegment,section,class
from #TempWithTotalDaysCount group by class,curriculumsegment,section
SET #query = 'SELECT [CurriculumSegment],Class,section,'+#cols+'
FROM (
SELECT [CurriculumSegment],
Class,section,PercentOfPresentDays,AttendanceDate FROM
#TempWithTotalDaysCount) AS up
PIVOT (MAX(PercentOfPresentDays) FOR AttendanceDate IN ('+#cols+')) AS pvt'
-- Query 2
EXEC (#query)
From the query 1 and query 2 results, I needs to do join and get the TotalCount, AbsentCount, PresentCount
in the result of query 2.
Is it possible to put "EXEC(#query)" in temp table ?
I tried but I'm getting invalid object. Because I cant give the column names as they come dynamically form pivot.
So,is there any other approach ?
Thank you.
Find out my result in below image:

Replace specials chars with HTML entities

I have the following in table TABLE
id content
-------------------------------------
1 Hellö world, I äm text
2 ènd there äré many more chars
3 that are speçial in my dat£base
I now need to export these records into HTML files, using bcp:
set #command = 'bcp "select [content] from [TABLE] where [id] = ' +
#id queryout +' + #filename + '.html" -S ' + #instance +
' -c -U ' + #username + ' -P ' + #password"
exec xp_cmdshell #command, no_ouput
To make the output look correct, I need to first replace all special characters with their respective HTML entities (pseudo)
insert into [#temp_html] ..
replace(replace([content], 'ö', 'ö'), 'ä', 'ä')
But by now, I have 30 nested replaces and it's starting to look insane.
After much searching, I found this post which uses a HTML conversion table but it is too advanced for me to understand:
The table does not list the special chars itself as they are in my text (ö, à etc) but UnicodeHex. Do I need to add them to the table to make the conversions that I need?
I am having trouble understanding how to update my script to replace all special chars. Can someone please show me a snippet of (pseudo) code?
One way to do that with a translation table is using a recursive cte to do the replaces, and one more cte to get only the last row of each translated value.
First, create and populate sample table (Please save us this step in your future questions):
DECLARE #T AS TABLE
(
id int,
content nvarchar(100)
)
INSERT INTO #T (id, content) VALUES
(1, 'Hellö world, I äm text'),
(2, 'ènd there äré many more chars'),
(3, 'that are speçial in my dat£base')
Then, create and populate the translation table (I don't know the HTML entities for these chars, so I've just used numbers [plus it's easier to see in the results]). Also, please note that this can be done using yet another cte in the chain.
DECLARE #Translations AS TABLE
(
str nchar(1),
replacement nvarchar(10)
)
INSERT INTO #Translations (str, replacement) VALUES
('ö', '-1-'),
('ä', '-2-'),
('è', '-3-'),
('ä', '-4-'),
('é', '-5-'),
('ç', '-6-'),
('£', '-7-')
Now, the first cte will do the replaces, and the second cte just adds a row_number so that for each id, the last value of lvl will get 1:
;WITH CTETranslations AS
(
SELECT id, content, 1 As lvl
FROM #T
UNION ALL
SELECT id, CAST(REPLACE(content, str, replacement) as nvarchar(100)), lvl+1
FROM CTETranslations
JOIN #Translations
ON content LIKE '%' + str + '%'
), cteNumberedTranslation AS
(
SELECT id, content, ROW_NUMBER() OVER(PARTITION BY Id ORDER BY lvl DESC) rn
FROM CTETranslations
)
Select from the second cte where rn = 1, I've joined the original table to show the source and translation side by side:
SELECT r.id, s.content, r.content
FROM #T s
JOIN cteNumberedTranslation r
ON s.Id = r.Id
WHERE rn = 1
ORDER BY Id
Results:
id content content
1 Hellö world, I äm text Hell-1- world, I -4-m text
2 ènd there äré many more chars -3-nd there -4-r-5- many more chars
3 that are speçial in my dat£base that are spe-6-ial in my dat-7-base
Please note that if your content have more that 100 special chars, you will need to add the maxrecursion 0 hint to the final select:
SELECT r.id, s.content, r.content
FROM #T s
JOIN cteNumberedTranslation r
ON s.Id = r.Id
WHERE rn = 1
ORDER BY Id
OPTION ( MAXRECURSION 0 );
See a live demo on rextester.

Dynamic Pivot with varying columns

I have a POA Code dynamic pivot that pulls data from a DX temp table and inserts the data into a temp POA table.
The issue I'm having is that there is a possibility of up to 35 different columns that can be returned. Depending on the month there could be 15 columns (POA1...POA15) or there could be all 35 columns (POA1...POA35). I join this dynamic pivot temp table on another patient table. My problem is, I need to show all 35 columns even if some of the columns do not exist in the temp POA table.
--Pivot DX POA Codes
DECLARE #POANAME VARCHAR(40)
SELECT #POAName = '##tmpPOA'
DECLARE #colsPOA NVARCHAR(2000)
SELECT #colsPOA = STUFF((SELECT DISTINCT TOP 100 PERCENT
'],[' + 'POA' + CAST(Dx.RowNum AS NVARCHAR)
FROM #tmpDX DX
ORDER BY '],[' + 'POA' + CAST(Dx.RowNum AS NVARCHAR)
FOR XML PATH ('')
),1,2,'') + ']'
DECLARE #queryPOA NVARCHAR(4000)
SET #queryPOA = 'N
SELECT
EncObjID,
'+
#colsPOA
+' INTO ' + POAName + '
FROM
(SELECT
Dx.EncObjID
,''POA'' + Dx.RowNum AS RowNum
,Dx.POAMne
FROM #tmpDx Dx
) p
PIVOT
(
MIN([POAMne])
FOR RowNum IN
( ' + #colsPOA + ' )
) AS pvt'
EXECUTE(#queryPOA)
I'm receiving an Invalid Column Name in my patient query because some of the columns don't exist in ##tmpPOA. I thought about creating a temp table called #tmpDxPOA and doing an insert (Insert Into #tmpDxPOA select * from ##tmpPOA), but that doesn't work (I receive a Column Name or number of supplied values does not match error).
Any thoughts on how to create all 35 columns even if there isn't any data? I don't care if they're null, I just need to have those place holders in the main patient query and it doesn't help that the number of columns returned varies every month.
With the help of #mxix I was able to come up with the following:
DECLARE #POASQL NVARCHAR(MAX)
SET #POASQL = N'INSERT INTO #tmpPOAFinal (EncObjID,'+#colsPOA+') SELECT * FROM ##tmpPOA'
EXECUTE(#POASQL)
I put this after the EXECUTE(#queryPOA) in my main query.
In order for this to work with Dynamic SQL the rows/colums need to exists more than zero times. Whether it be for one or more patient. I would try to fan out the number of POA possibilities right off the bat and then left outer join to get the actual values back.
IF OBJECT_ID('tempdb..#tmpPOA') IS NOT NULL DROP TABLE #tmpPOA
CREATE TABLE #tmpPOA (POA varchar(10))
IF OBJECT_ID('tempdb..#tmpPatient') IS NOT NULL DROP TABLE #tmpPatient
CREATE TABLE #tmpPatient (Patient varchar(15))
INSERT INTO #tmpPatient VALUES ('ABC123'),('ABC456'),('ABC789')
DECLARE #POAFlag as INT = 0
WHILE #POAFlag <36
BEGIN
INSERT INTO #tmpPOA
VALUES('POA' +CONVERT(varchar,#POAFlag))
SET #POAFlag = #POAFlag + 1
END
SELECT * FROM #tmpPOA
CROSS JOIN #tmpPatient
This should fan out all of the possibilities of the 35DXCodes for you to get their POA flag.

Paging, sorting and filtering in a stored procedure (SQL Server)

I was looking at different ways of writing a stored procedure to return a "page" of data. This was for use with the ASP ObjectDataSource, but it could be considered a more general problem.
The requirement is to return a subset of the data based on the usual paging parameters; startPageIndex and maximumRows, but also a sortBy parameter to allow the data to be sorted. Also there are some parameters passed in to filter the data on various conditions.
One common way to do this seems to be something like this:
[Method 1]
;WITH stuff AS (
SELECT
CASE
WHEN #SortBy = 'Name' THEN ROW_NUMBER() OVER (ORDER BY Name)
WHEN #SortBy = 'Name DESC' THEN ROW_NUMBER() OVER (ORDER BY Name DESC)
WHEN #SortBy = ...
ELSE ROW_NUMBER() OVER (ORDER BY whatever)
END AS Row,
.,
.,
.,
FROM Table1
INNER JOIN Table2 ...
LEFT JOIN Table3 ...
WHERE ... (lots of things to check)
)
SELECT *
FROM stuff
WHERE (Row > #startRowIndex)
AND (Row <= #startRowIndex + #maximumRows OR #maximumRows <= 0)
ORDER BY Row
One problem with this is that it doesn't give the total count and generally we need another stored procedure for that. This second stored procedure has to replicate the parameter list and the complex WHERE clause. Not nice.
One solution is to append an extra column to the final select list, (SELECT COUNT(*) FROM stuff) AS TotalRows. This gives us the total but repeats it for every row in the result set, which is not ideal.
[Method 2]
An interesting alternative is given here (https://web.archive.org/web/20211020111700/https://www.4guysfromrolla.com/articles/032206-1.aspx) using dynamic SQL. He reckons that the performance is better because the CASE statement in the first solution drags things down. Fair enough, and this solution makes it easy to get the totalRows and slap it into an output parameter. But I hate coding dynamic SQL. All that 'bit of SQL ' + STR(#parm1) +' bit more SQL' gubbins.
[Method 3]
The only way I can find to get what I want, without repeating code which would have to be synchronized, and keeping things reasonably readable is to go back to the "old way" of using a table variable:
DECLARE #stuff TABLE (Row INT, ...)
INSERT INTO #stuff
SELECT
CASE
WHEN #SortBy = 'Name' THEN ROW_NUMBER() OVER (ORDER BY Name)
WHEN #SortBy = 'Name DESC' THEN ROW_NUMBER() OVER (ORDER BY Name DESC)
WHEN #SortBy = ...
ELSE ROW_NUMBER() OVER (ORDER BY whatever)
END AS Row,
.,
.,
.,
FROM Table1
INNER JOIN Table2 ...
LEFT JOIN Table3 ...
WHERE ... (lots of things to check)
SELECT *
FROM stuff
WHERE (Row > #startRowIndex)
AND (Row <= #startRowIndex + #maximumRows OR #maximumRows <= 0)
ORDER BY Row
(Or a similar method using an IDENTITY column on the table variable).
Here I can just add a SELECT COUNT on the table variable to get the totalRows and put it into an output parameter.
I did some tests and with a fairly simple version of the query (no sortBy and no filter), method 1 seems to come up on top (almost twice as quick as the other 2). Then I decided to test probably I needed the complexity and I needed the SQL to be in stored procedures. With this I get method 1 taking nearly twice as long as the other 2 methods. Which seems strange.
Is there any good reason why I shouldn't spurn CTEs and stick with method 3?
UPDATE - 15 March 2012
I tried adapting Method 1 to dump the page from the CTE into a temporary table so that I could extract the TotalRows and then select just the relevant columns for the resultset. This seemed to add significantly to the time (more than I expected). I should add that I'm running this on a laptop with SQL Server Express 2008 (all that I have available) but still the comparison should be valid.
I looked again at the dynamic SQL method. It turns out I wasn't really doing it properly (just concatenating strings together). I set it up as in the documentation for sp_executesql (with a parameter description string and parameter list) and it's much more readable. Also this method runs fastest in my environment. Why that should be still baffles me, but I guess the answer is hinted at in Hogan's comment.
I would most likely split the #SortBy argument into two, #SortColumn and #SortDirection, and use them like this:
…
ROW_NUMBER() OVER (
ORDER BY CASE #SortColumn
WHEN 'Name' THEN Name
WHEN 'OtherName' THEN OtherName
…
END *
CASE #SortDirection
WHEN 'DESC' THEN -1
ELSE 1
END
) AS Row
…
And this is how the TotalRows column could be defined (in the main select):
…
COUNT(*) OVER () AS TotalRows
…
I would definitely want to do a combination of a temp table and NTILE for this sort of approach.
The temp table will allow you to do your complicated series of conditions just once. Because you're only storing the pieces you care about, it also means that when you start doing selects against it further in the procedure, it should have a smaller overall memory usage than if you ran the condition multiple times.
I like NTILE() for this better than ROW_NUMBER() because it's doing the work you're trying to accomplish for you, rather than having additional where conditions to worry about.
The example below is one based off a similar query I'm using as part of a research query; I have an ID I can use that I know will be unique in the results. Using an ID that was an identity column would also be appropriate here, though.
--DECLARES here would be stored procedure parameters
declare #pagesize int, #sortby varchar(25), #page int = 1;
--Create temp with all relevant columns; ID here could be an identity PK to help with paging query below
create table #temp (id int not null primary key clustered, status varchar(50), lastname varchar(100), startdate datetime);
--Insert into #temp based off of your complex conditions, but with no attempt at paging
insert into #temp
(id, status, lastname, startdate)
select id, status, lastname, startdate
from Table1 ...etc.
where ...complicated conditions
SET #pagesize = 50;
SET #page = 5;--OR CAST(#startRowIndex/#pagesize as int)+1
SET #sortby = 'name';
--Only use the id and count to use NTILE
;with paging(id, pagenum, totalrows) as
(
select id,
NTILE((SELECT COUNT(*) cnt FROM #temp)/#pagesize) OVER(ORDER BY CASE WHEN #sortby = 'NAME' THEN lastname ELSE convert(varchar(10), startdate, 112) END),
cnt
FROM #temp
cross apply (SELECT COUNT(*) cnt FROM #temp) total
)
--Use the id to join back to main select
SELECT *
FROM paging
JOIN #temp ON paging.id = #temp.id
WHERE paging.pagenum = #page
--Don't need the drop in the procedure, included here for rerunnability
drop table #temp;
I generally prefer temp tables over table variables in this scenario, largely so that there are definite statistics on the result set you have. (Search for temp table vs table variable and you'll find plenty of examples as to why)
Dynamic SQL would be most useful for handling the sorting method. Using my example, you could do the main query in dynamic SQL and only pull the sort method you want to pull into the OVER().
The example above also does the total in each row of the return set, which as you mentioned was not ideal. You could, instead, have a #totalrows output variable in your procedure and pull it as well as the result set. That would save you the CROSS APPLY that I'm doing above in the paging CTE.
I would create one procedure to stage, sort, and paginate (using NTILE()) a staging table; and a second procedure to retrieve by page. This way you don't have to run the entire main query for each page.
This example queries AdventureWorks.HumanResources.Employee:
--------------------------------------------------------------------------
create procedure dbo.EmployeesByMartialStatus
#MaritalStatus nchar(1)
, #sort varchar(20)
as
-- Init staging table
if exists(
select 1 from sys.objects o
inner join sys.schemas s on s.schema_id=o.schema_id
and s.name='Staging'
and o.name='EmployeesByMartialStatus'
where type='U'
)
drop table Staging.EmployeesByMartialStatus;
-- Populate staging table with sort value
with s as (
select *
, sr=ROW_NUMBER()over(order by case #sort
when 'NationalIDNumber' then NationalIDNumber
when 'ManagerID' then ManagerID
-- plus any other sort conditions
else EmployeeID end)
from AdventureWorks.HumanResources.Employee
where MaritalStatus=#MaritalStatus
)
select *
into #temp
from s;
-- And now pages
declare #RowCount int; select #rowCount=COUNT(*) from #temp;
declare #PageCount int=ceiling(#rowCount/20); --assuming 20 lines/page
select *
, Page=NTILE(#PageCount)over(order by sr)
into Staging.EmployeesByMartialStatus
from #temp;
go
--------------------------------------------------------------------------
-- procedure to retrieve selected pages
create procedure EmployeesByMartialStatus_GetPage
#page int
as
declare #MaxPage int;
select #MaxPage=MAX(Page) from Staging.EmployeesByMartialStatus;
set #page=case when #page not between 1 and #MaxPage then 1 else #page end;
select EmployeeID,NationalIDNumber,ContactID,LoginID,ManagerID
, Title,BirthDate,MaritalStatus,Gender,HireDate,SalariedFlag,VacationHours,SickLeaveHours
, CurrentFlag,rowguid,ModifiedDate
from Staging.EmployeesByMartialStatus
where Page=#page
GO
--------------------------------------------------------------------------
-- Usage
-- Load staging
exec dbo.EmployeesByMartialStatus 'M','NationalIDNumber';
-- Get pages 1 through n
exec dbo.EmployeesByMartialStatus_GetPage 1;
exec dbo.EmployeesByMartialStatus_GetPage 2;
-- ...etc (this would actually be a foreach loop, but that detail is omitted for brevity)
GO
I use this method of using EXEC():
-- SP parameters:
-- #query: Your query as an input parameter
-- #maximumRows: As number of rows per page
-- #startPageIndex: As number of page to filter
-- #sortBy: As a field name or field names with supporting DESC keyword
DECLARE #query nvarchar(max) = 'SELECT * FROM sys.Objects',
#maximumRows int = 8,
#startPageIndex int = 3,
#sortBy as nvarchar(100) = 'name Desc'
SET #query = ';WITH CTE AS (' + #query + ')' +
'SELECT *, (dt.pagingRowNo - 1) / ' + CAST(#maximumRows as nvarchar(10)) + ' + 1 As pagingPageNo' +
', pagingCountRow / ' + CAST(#maximumRows as nvarchar(10)) + ' As pagingCountPage ' +
', (dt.pagingRowNo - 1) % ' + CAST(#maximumRows as nvarchar(10)) + ' + 1 As pagingRowInPage ' +
'FROM ( SELECT *, ROW_NUMBER() OVER (ORDER BY ' + #sortBy + ') As pagingRowNo, COUNT(*) OVER () AS pagingCountRow ' +
'FROM CTE) dt ' +
'WHERE (dt.pagingRowNo - 1) / ' + CAST(#maximumRows as nvarchar(10)) + ' + 1 = ' + CAST(#startPageIndex as nvarchar(10))
EXEC(#query)
At result-set after query result columns:
Note:
I add some extra columns that you can remove them:
pagingRowNo : The row number
pagingCountRow : The total number of rows
pagingPageNo : The current page number
pagingCountPage : The total number of pages
pagingRowInPage : The row number that started with 1 in this page

Resources