SqlServer Random Data Generation Observation - sql-server

I have a question on why the output of these two queries differ. I would have expected them to work the same.
Query 1:
declare #cache table(originalValue nvarchar(255), obfuscateValue nvarchar(255));
declare #table1 table(c char(1));
declare #i1 int;
set #i1 = ASCII('0');
while #i1 <= ASCII('9')
begin
insert into #table1 (c)
select (CHAR(#i1))
set #i1 = #i1 +1;
end
insert into #cache (originalValue, obfuscateValue)
select [firstname],
(select top 1 c from #table1 order by NEWID()) +
(select top 1 c from #table1 order by NEWID())
from Customer
where [firstname] is not null
select * from #cache;
Query 2:
declare #cache table(originalValue nvarchar(255), obfuscateValue nvarchar(255));
declare #table1 table(c char(1));
declare #i1 int;
set #i1 = ASCII('0');
while #i1 <= ASCII('9')
begin
insert into #table1 (c)
select (CHAR(#i1))
set #i1 = #i1 +1;
end
insert into #cache (originalValue)
select [firstname]
from Customer
where [firstname] is not null
update c
set c.obfuscateValue = t.Value
from #cache c
join
(
select originalValue,
(
(select top 1 c from #table1 order by NEWID()) +
(select top 1 c from #table1 order by NEWID())
) as Value
from #cache
) t on t.originalValue = c.originalValue
select * from #cache;
They should do the same, but first query returns following results:
Jonathon 73
Everett 73
Janet 73
Andy 73
Shauna 73
And second:
Jonathon 82
Everett 40
Janet 68
Andy 79
Shauna 29
As you noticed, the second column in second result has different values, while first - same values.
It looks like in first query the
(select top 1 c from #table1 order by NEWID()) +
(select top 1 c from #table1 order by NEWID())
is called only once.
Can someone explain this mystery?

I think random values can be generated in another way.
This is how to generated [a-zA-Z]{3,6}
declare #min int, #max int;
declare #alpha varchar(max)
set #min = 3;
set #max = 6;
set #alpha = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
declare #cache table(originalValue nvarchar(255), obfuscateValue nvarchar(255));
insert into #cache (originalValue, obfuscateValue)
select [firstname], LEFT(t.Value, case when t.maxLen < #min then #min else t.maxLen end)
from Customer
join
(
select ABS(CHECKSUM(NEWID()))%#max + 1 as maxLen,
SUBSTRING(#alpha, ABS(CHECKSUM(NEWID()))%LEN(#alpha) + 1, 1) +
SUBSTRING(#alpha, ABS(CHECKSUM(NEWID()))%LEN(#alpha) + 1, 1) +
SUBSTRING(#alpha, ABS(CHECKSUM(NEWID()))%LEN(#alpha) + 1, 1) +
SUBSTRING(#alpha, ABS(CHECKSUM(NEWID()))%LEN(#alpha) + 1, 1) +
SUBSTRING(#alpha, ABS(CHECKSUM(NEWID()))%LEN(#alpha) + 1, 1) +
SUBSTRING(#alpha, ABS(CHECKSUM(NEWID()))%LEN(#alpha) + 1, 1) as Value
)t on t.Value is not null
where [firstname] is not null
select * from #cache;

One line?
SELECT
RIGHT( --number of zeros to match expected max length. Or use REPLICATE.
'000000' + CAST(
--The 2 newid() expression means we'll get a larger number
--less chance of using leading static zeroes
CAST(CHECKSUM(NEWD_ID()) as bigint) * CAST(CHECKSUM(NEWD_ID()) as bigint)
as varchar(30))
--The 3 gives us the desired mask. Currently 3 digits.
, 3)

You are correct in your assumption that the first query is only running the ‘select top’ once. The behavior is happening because of how the optimizer chose to optimize the query. It decided because the subqueries (the select top queries) are self-contained and are not correlated with the outside select query it uses a Tablespool (Lazy Spool) operator in the execution plan. This causes the select top value to be placed in the tempdb for reuse.
Because the optimizer chooses to use a Nested Loops operator to bring all the data together no rebinding is needed, the spooled value is used instead of reapplying the queries for each input outer row.
During the second query the optimizer chose not to use a Tablespool operator (I believe do to fact the input table being from tempdb). So you have the select top subqueries being reapplied for each input row from the temporary table.
If needed, you may be able to use a table/query hints if you want to force the execution plan to perform as desired.

Related

Substring is slow with while loop in SQL Server

One of my table column stores ~650,000 characters (each value of the column contains entire table). I know its bad design however, Client will not be able to change it.
I am tasked to convert the column into multiple columns.
I chose to use dbo.DelimitedSplit8K function
Unfortunately, it can only handle 8k characters at max.
So I decided to split the column into 81 8k batches using while loop and store the same in a variable table (temp or normal table made no improvement)
DECLARE #tab1 table ( serialnumber int, etext nvarchar(1000))
declare #scriptquan int = (select MAX(len (errortext)/8000) from mytable)
DECLARE #Counter INT
DECLARE #A bigint = 1
DECLARE #B bigint = 8000
SET #Counter=1
WHILE ( #Counter <= #scriptquan + 1)
BEGIN
insert into #tab1 select ItemNumber, Item from dbo.mytable cross apply dbo.DelimitedSplit8K(substring(errortext, #A, #B), CHAR(13)+CHAR(10))
SET #A = #A + 8000
SET #B = #B + 8000
SET #Counter = #Counter + 1
END
This followed by using below code
declare #tab2 table (Item nvarchar(max),itemnumber int, Colseq varchar(10)) -- declare table variable
;with cte as (
select [etext] ,ItemNumber, Item from #tab1 -- insert table name
cross apply dbo.DelimitedSplit8K(etext,' ')) -- insert table columns name that contains text
insert into #tab2 Select Item,itemnumber, 'a'+ cast (ItemNumber as varchar) colseq
from cte -- insert values to table variable
;WITH Tbl(item, colseq) AS(
select item, colseq from #tab2
),
CteRn AS(
SELECT item, colseq,
Rn = ROW_NUMBER() OVER(PARTITION BY colseq ORDER BY colseq)
FROM Tbl
)
SELECT
a1 Time,a2 Number,a3 Type,a4 Remarks
FROM CteRn r
PIVOT(
MAX(item)
FOR colseq IN(a1,a2,a3,a4)
)p
where a3 = 'error'
gives the desired output. However, just the loop takes 15 minutes to complete and overall query completes by 27 minutes. Is there any way I can make it faster? Total row count in my table is 2. So I don't think Index can help.
Client uses Azure SQL Database so I can't choose PowerShell or Python to accomplish this either.
Please let me know if more information is needed. I tried my best to mention everything I could.

Remove some characters from string sql [duplicate]

I've got dirty data in a column with variable alpha length. I just want to strip out anything that is not 0-9.
I do not want to run a function or proc. I have a script that is similar that just grabs the numeric value after text, it looks like this:
Update TableName
set ColumntoUpdate=cast(replace(Columnofdirtydata,'Alpha #','') as int)
where Columnofdirtydata like 'Alpha #%'
And ColumntoUpdate is Null
I thought it would work pretty good until I found that some of the data fields I thought would just be in the format Alpha # 12345789 are not.
Examples of data that needs to be stripped
AB ABCDE # 123
ABCDE# 123
AB: ABC# 123
I just want the 123. It is true that all data fields do have the # prior to the number.
I tried substring and PatIndex, but I'm not quite getting the syntax correct or something. Anyone have any advice on the best way to address this?
See this blog post on extracting numbers from strings in SQL Server. Below is a sample using a string in your example:
DECLARE #textval NVARCHAR(30)
SET #textval = 'AB ABCDE # 123'
SELECT LEFT(SUBSTRING(#textval, PATINDEX('%[0-9.-]%', #textval), 8000),
PATINDEX('%[^0-9.-]%', SUBSTRING(#textval, PATINDEX('%[0-9.-]%', #textval), 8000) + 'X') -1)
Here is an elegant solution if your server supports the TRANSLATE function (on sql server it's available on sql server 2017+ and also sql azure).
First, it replaces any non numeric characters with a # character.
Then, it removes all # characters.
You may need to add additional characters that you know may be present in the second parameter of the TRANSLATE call.
select REPLACE(TRANSLATE([Col], 'abcdefghijklmnopqrstuvwxyz+()- ,#+', '##################################'), '#', '')
You can use stuff and patindex.
stuff(Col, 1, patindex('%[0-9]%', Col)-1, '')
SQL Fiddle
This works well for me:
CREATE FUNCTION [dbo].[StripNonNumerics]
(
#Temp varchar(255)
)
RETURNS varchar(255)
AS
Begin
Declare #KeepValues as varchar(50)
Set #KeepValues = '%[^0-9]%'
While PatIndex(#KeepValues, #Temp) > 0
Set #Temp = Stuff(#Temp, PatIndex(#KeepValues, #Temp), 1, '')
Return #Temp
End
Then call the function like so to see the original something next to the sanitized something:
SELECT Something, dbo.StripNonNumerics(Something) FROM TableA
In case if there are some characters possible between digits (e.g. thousands separators), you may try following:
declare #table table (DirtyCol varchar(100))
insert into #table values
('AB ABCDE # 123')
,('ABCDE# 123')
,('AB: ABC# 123')
,('AB#')
,('AB # 1 000 000')
,('AB # 1`234`567')
,('AB # (9)(876)(543)')
;with tally as (select top (100) N=row_number() over (order by ##spid) from sys.all_columns),
data as (
select DirtyCol, Col
from #table
cross apply (
select (select C + ''
from (select N, substring(DirtyCol, N, 1) C from tally where N<=datalength(DirtyCol)) [1]
where C between '0' and '9'
order by N
for xml path(''))
) p (Col)
where p.Col is not NULL
)
select DirtyCol, cast(Col as int) IntCol
from data
Output is:
DirtyCol IntCol
--------------------- -------
AB ABCDE # 123 123
ABCDE# 123 123
AB: ABC# 123 123
AB # 1 000 000 1000000
AB # 1`234`567 1234567
AB # (9)(876)(543) 9876543
For update, add ColToUpdate to select list of the data cte:
;with num as (...),
data as (
select ColToUpdate, /*DirtyCol, */Col
from ...
)
update data
set ColToUpdate = cast(Col as int)
CREATE FUNCTION FN_RemoveNonNumeric (#Input NVARCHAR(512))
RETURNS NVARCHAR(512)
AS
BEGIN
DECLARE #Trimmed NVARCHAR(512)
SELECT #Trimmed = #Input
WHILE PATINDEX('%[^0-9]%', #Trimmed) > 0
SELECT #Trimmed = REPLACE(#Trimmed, SUBSTRING(#Trimmed, PATINDEX('%[^0-9]%', #Trimmed), 1), '')
RETURN #Trimmed
END
GO
SELECT dbo.FN_RemoveNonNumeric('ABCDE# 123')
Pretty late to the party, I found the following which I though worked brilliantialy.. if anyone is still looking
SELECT
(SELECT CAST(CAST((
SELECT SUBSTRING(FieldToStrip, Number, 1)
FROM master..spt_values
WHERE Type='p' AND Number <= LEN(FieldToStrip) AND
SUBSTRING(FieldToStrip, Number, 1) LIKE '[0-9]' FOR XML Path(''))
AS xml) AS varchar(MAX)))
FROM
SourceTable
Here's a version which pulls all digits from a string; i.e. given I'm 35 years old; I was born in 1982. The average family has 2.4 children. this would return 35198224. i.e. it's good where you've got numeric data which may have been formatted as a code (e.g. #123,456,789 / 123-00005), but isn't appropriate if you're looking to pull out specific numbers (i.e. as opposed to digits / just the numeric characters) from the text. Also it only handles digits; so won't return negative signs (-) or periods .).
declare #table table (id bigint not null identity (1,1), data nvarchar(max))
insert #table (data)
values ('hello 123 its 45613 then') --outputs: 12345613
,('1 some other string 98 example 4') --outputs: 1984
,('AB ABCDE # 123') --outputs: 123
,('ABCDE# 123') --outputs: 123
,('AB: ABC# 123') --outputs: 123
; with NonNumerics as (
select id
, data original
--the below line replaces all digits with blanks
, replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(data,'0',''),'1',''),'2',''),'3',''),'4',''),'5',''),'6',''),'7',''),'8',''),'9','') nonNumeric
from #table
)
--each iteration of the below CTE removes another non-numeric character from the original string, putting the result into the numerics column
, Numerics as (
select id
, replace(original, substring(nonNumeric,1,1), '') numerics
, replace(nonNumeric, substring(nonNumeric,1,1), '') charsToreplace
, len(replace(nonNumeric, substring(nonNumeric,1,1), '')) charsRemaining
from NonNumerics
union all
select id
, replace(numerics, substring(charsToreplace,1,1), '') numerics
, replace(charsToreplace, substring(charsToreplace,1,1), '') charsToreplace
, len(replace(charsToreplace, substring(charsToreplace,1,1), '')) charsRemaining
from Numerics
where charsRemaining > 0
)
--we select only those strings with `charsRemaining=0`; i.e. the rows for which all non-numeric characters have been removed; there should be 1 row returned for every 1 row in the original data set.
select * from Numerics where charsRemaining = 0
This code works by removing all the digits (i.e. the characters we want) from a the given strings by replacing them with blanks. Then it goes through the original string (which includes the digits) removing all of the characters that were left (i.e. the non-numeric characters), thus leaving only the digits.
The reason we do this in 2 steps, rather than just removing all non-numeric characters in the first place is there are only 10 digits, whilst there are a huge number of possible characters; so replacing that small list is relatively fast; then gives us a list of those non-numeric characters which actually exist in the string, so we can then replace that small set.
The method makes use of recursive SQL, using common table expressions (CTEs).
To add on to Ken's answer, this handles commas and spaces and parentheses
--Handles parentheses, commas, spaces, hyphens..
declare #table table (c varchar(256))
insert into #table
values
('This is a test 111-222-3344'),
('Some Sample Text (111)-222-3344'),
('Hello there 111222 3344 / How are you?'),
('Hello there 111 222 3344 ? How are you?'),
('Hello there 111 222 3344. How are you?')
select
replace(LEFT(SUBSTRING(replace(replace(replace(replace(replace(c,'(',''),')',''),'-',''),' ',''),',',''), PATINDEX('%[0-9.-]%', replace(replace(replace(replace(replace(c,'(',''),')',''),'-',''),' ',''),',','')), 8000),
PATINDEX('%[^0-9.-]%', SUBSTRING(replace(replace(replace(replace(replace(c,'(',''),')',''),'-',''),' ',''),',',''), PATINDEX('%[0-9.-]%', replace(replace(replace(replace(replace(c,'(',''),')',''),'-',''),' ',''),',','')), 8000) + 'X') -1),'.','')
from #table
Create function fn_GetNumbersOnly(#pn varchar(100))
Returns varchar(max)
AS
BEGIN
Declare #r varchar(max) ='', #len int ,#c char(1), #x int = 0
Select #len = len(#pn)
while #x <= #len
begin
Select #c = SUBSTRING(#pn,#x,1)
if ISNUMERIC(#c) = 1 and #c <> '-'
Select #r = #r + #c
Select #x = #x +1
end
return #r
End
In your case It seems like the # will always be after teh # symbol so using CHARINDEX() with LTRIM() and RTRIM() would probably perform the best. But here is an interesting method of getting rid of ANY non digit. It utilizes a tally table and table of digits to limit which characters are accepted then XML technique to concatenate back to a single string without the non-numeric characters. The neat thing about this technique is it could be expanded to included ANY Allowed characters and strip out anything that is not allowed.
DECLARE #ExampleData AS TABLE (Col VARCHAR(100))
INSERT INTO #ExampleData (Col) VALUES ('AB ABCDE # 123'),('ABCDE# 123'),('AB: ABC# 123')
DECLARE #Digits AS TABLE (D CHAR(1))
INSERT INTO #Digits (D) VALUES ('0'),('1'),('2'),('3'),('4'),('5'),('6'),('7'),('8'),('9')
;WITH cteTally AS (
SELECT
I = ROW_NUMBER() OVER (ORDER BY (SELECT NULL))
FROM
#Digits d10
CROSS APPLY #Digits d100
--add more cross applies to cover longer fields this handles 100
)
SELECT *
FROM
#ExampleData e
OUTER APPLY (
SELECT CleansedPhone = CAST((
SELECT TOP 100
SUBSTRING(e.Col,t.I,1)
FROM
cteTally t
INNER JOIN #Digits d
ON SUBSTRING(e.Col,t.I,1) = d.D
WHERE
I <= LEN(e.Col)
ORDER BY
t.I
FOR XML PATH('')) AS VARCHAR(100))) o
Declare #MainTable table(id int identity(1,1),TextField varchar(100))
INSERT INTO #MainTable (TextField)
VALUES
('6B32E')
declare #i int=1
Declare #originalWord varchar(100)=''
WHile #i<=(Select count(*) from #MainTable)
BEGIN
Select #originalWord=TextField from #MainTable where id=#i
Declare #r varchar(max) ='', #len int ,#c char(1), #x int = 0
Select #len = len(#originalWord)
declare #pn varchar(100)=#originalWord
while #x <= #len
begin
Select #c = SUBSTRING(#pn,#x,1)
if(#c!='')
BEGIN
if ISNUMERIC(#c) = 0 and #c <> '-'
BEGIN
Select #r = cast(#r as varchar) + cast(replace((SELECT ASCII(#c)-64),'-','') as varchar)
end
ELSE
BEGIN
Select #r = #r + #c
END
END
Select #x = #x +1
END
Select #r
Set #i=#i+1
END
I have created a function for this
Create FUNCTION RemoveCharacters (#text varchar(30))
RETURNS VARCHAR(30)
AS
BEGIN
declare #index as int
declare #newtexval as varchar(30)
set #index = (select PATINDEX('%[A-Z.-/?]%', #text))
if (#index =0)
begin
return #text
end
else
begin
set #newtexval = (select STUFF ( #text , #index , 1 , '' ))
return dbo.RemoveCharacters(#newtexval)
end
return 0
END
GO
Here is the answer:
DECLARE #t TABLE (tVal VARCHAR(100))
INSERT INTO #t VALUES('123')
INSERT INTO #t VALUES('123S')
INSERT INTO #t VALUES('A123,123')
INSERT INTO #t VALUES('a123..A123')
;WITH cte (original, tVal, n)
AS
(
SELECT t.tVal AS original,
LOWER(t.tVal) AS tVal,
65 AS n
FROM #t AS t
UNION ALL
SELECT tVal AS original,
CAST(REPLACE(LOWER(tVal), LOWER(CHAR(n)), '') AS VARCHAR(100)),
n + 1
FROM cte
WHERE n <= 90
)
SELECT t1.tVal AS OldVal,
t.tval AS NewVal
FROM (
SELECT original,
tVal,
ROW_NUMBER() OVER(PARTITION BY tVal + original ORDER BY original) AS Sl
FROM cte
WHERE PATINDEX('%[a-z]%', tVal) = 0
) t
INNER JOIN #t t1
ON t.original = t1.tVal
WHERE t.sl = 1
You can create SQL CLR scalar function in order to be able to use regular expressions like replace patterns.
Here you can find example of how to create such function.
Having such function will solve the issue with just the following lines:
SELECT [dbo].[fn_Utils_RegexReplace] ('AB ABCDE # 123', '[^0-9]', '');
SELECT [dbo].[fn_Utils_RegexReplace] ('ABCDE# 123', '[^0-9]', '');
SELECT [dbo].[fn_Utils_RegexReplace] ('AB: ABC# 123', '[^0-9]', '');
More important, you will be able to solve more complex issues as the regular expressions will bring a whole new world of options directly in your T-SQL statements.
Use this:
REPLACE(TRANSLATE(SomeString, REPLACE(TRANSLATE(SomeString, '0123456789', '##########'), '#', ''), REPLICATE('#', LEN(REPLACE(TRANSLATE(SomeString, '0123456789', '##########'), '#', '') + 'x') - 1)), '#', '')
Demo:
DROP TABLE IF EXISTS #MyTempTable;
CREATE TABLE #MyTempTable (SomeString VARCHAR(255));
INSERT INTO #MyTempTable
VALUES ('ssss123ssg99d362sdg')
, ('hey 62q&*^(n43')
, (NULL)
, ('')
, ('hi')
, ('123');
SELECT SomeString
, REPLACE(TRANSLATE(SomeString, REPLACE(TRANSLATE(SomeString, '0123456789', '##########'), '#', ''), REPLICATE('#', LEN(REPLACE(TRANSLATE(SomeString, '0123456789', '##########'), '#', '') + 'x') - 1)), '#', '')
FROM #MyTempTable;
DROP TABLE IF EXISTS #MyTempTable;
Results:
SomeString
(No column name)
ssss123ssg99d362sdg
12399362
hey62q&*^(n43
6243
NULL
NULL
hi
123
123
While the OP wanted to "strip out anything that is not 0-9", the post is also tagged with "substring" and "patindex", and the OP mentioned the concern "not quite getting the syntax correct or something". To address that the requirements note that "all data fields do have the # prior to the number" and to provide an answer that addresses the challenges with substring/patindex, consider the following:
/* A sample select */
;WITH SampleValues AS
( SELECT 'AB ABCDE # 123' [Columnofdirtydata]
UNION ALL SELECT 'AB2: ABC# 123')
SELECT
s.Columnofdirtydata,
f1.pos1,
'['+ f2.substr +']' [InspectOutput]
FROM
SampleValues s
CROSS APPLY (SELECT PATINDEX('%# %',s.Columnofdirtydata) [pos1]) f1
CROSS APPLY (SELECT SUBSTRING(s.Columnofdirtydata, f1.pos1 + LEN('#-'),LEN(s.Columnofdirtydata)) [substr]) f2
/* Using update scenario from OP */
UPDATE t1
SET t1.Columntoupdate = CAST(f2.substr AS INT)
FROM
TableName t1
CROSS APPLY (SELECT PATINDEX('%# %',t1.Columnofdirtydata) [pos1]) f1
CROSS APPLY (SELECT SUBSTRING(t1.Columnofdirtydata, f1.pos1 + LEN('#-'),LEN(t1.Columnofdirtydata)) [substr]) f2
Note that my syntax advice for patindex/substring, is to:
consider using APPLY as a way to temporarily alias results from one function for use as parameters in the next. It's not uncommon to (in ETL, for example) need to parse out parameter/position-based substrings in an updatable column of a staging table. If you need to "debug" and potentially fix some parsing logic, this style will help.
consider using LEN('PatternSample') in your substring logic, to account for reusing this pattern or adjusting it when your source data changes (instead of "+ 1"
SUBSTRING() requires a length parameter, but it can be greater than the length of the string. Therefore, if you are getting "the rest of the string" after the pattern, you can just use "The source length"
DECLARE #STR VARCHAR(400)
DECLARE #specialchars VARCHAR(50) = '%[~,#,#,$,%,&,*,(,),!^?:]%'
SET #STR = '1, 45 4,3 68.00-'
WHILE PATINDEX( #specialchars, #STR ) > 0
---Remove special characters using Replace function
SET #STR = Replace(Replace(REPLACE( #STR, SUBSTRING( #STR, PATINDEX( #specialchars, #STR ), 1 ),''),'-',''), ' ','')
SELECT #STR
SELECT REGEXP_REPLACE( col, '[^[:digit:]]', '' ) AS new_col FROM my_table

How to insert multiple rows into a table based on a range of numbers

I have to insert a specific number of rows into a SQL Server table.
DECLARE #val AS INT = 20,
#val2 AS VARCHAR(50),
#Date AS DATETIME = CONVERT(DATETIME,'02-05-2016'),
#i AS INT = 0
SET #val2 = 'abc'
DECLARE #tbl TABLE
(
[ID] [int] IDENTITY(1,1) NOT NULL,
[val2] VARCHAR(50) NULL,
[datum] [datetime] NULL
)
--INSERT INTO #tbl
SELECT #val2, DATEADD(DAY, #i, #Date)
UNION ALL
SELECT #val2, DATEADD(DAY, #i, #Date)
In this query, I have to insert dates starting from a given date till the number of value assigned to the variable '#val'. So, in this case, 20 rows need to be inserted into the table starting from '02-05-2016' and then date increasing 1 day for each row.
How can I do it in a single statement without any looping or multiple insert statements?
You can use a numbers table if you have one, use master.dbo.spt_values if you want one that has values till 2048, or create one of your own. In this case, you could use master.dbo.spt_values:
DECLARE #val AS INT=20, #val2 AS VARCHAR(50);
DECLARE #Date AS DATETIME = CONVERT(DATETIME,'02-05-2016');
SET #val2 = 'abc'
INSERT INTO dbo.YourTable
SELECT #val2, DATEADD(DAY,number,#Date)
FROM master.dbo.spt_values
WHERE type = 'P'
AND number <= #val;
Though since this starts at zero, you'll get 21 rows as a result
Besides the detailed answer I pointed to in my comment, this is the idea in short:
DECLARE #start INT=0;
DECLARE #end INT=19; --0 to 19 are 20 days
DECLARE #StartDate DATE={d'2016-01-01'};
--Create a List of up to 1.000.000.000 rows on the fly
--This is limited by start and end parameter
;WITH x AS(SELECT 1 AS N FROM(VALUES(1),(1),(1),(1),(1),(1),(1),(1),(1),(1)) AS tbl(N))--10^1
,N3 AS (SELECT 1 AS N FROM x CROSS JOIN x AS N2 CROSS JOIN x N3) --10^3
,Tally AS(SELECT TOP(#end-#start +1) ROW_NUMBER() OVER(ORDER BY(SELECT NULL)) + #start -1 AS Nr FROM N3
CROSS JOIN N3 N6 CROSS JOIN N3 AS N9)
--INSERT INTO your_table
SELECT #val2 --your #val2 here as a constant value
,DATEADD(DAY,Nr,#StartDate)
FROM Tally
You could use a recursive CTE.
DECLARE #i INT = 1
, #m INT = 19
, #d DATETIME2 = '2016-05-02';
WITH i AS (
SELECT 0 AS increment
UNION ALL
SELECT i.increment + #i
FROM i
WHERE i.increment < #m
)
SELECT i.increment
, DATEADD(DAY, i.increment, #d)
FROM i
OPTION (MAXRECURSION 100);
Note the OPTION (MAXRECUSION 100) hint at the bottom, which is not strictly necessary but I have included it to illustrate how it works. By default, there is a limit of 100 results using this method, so without this statement and if #m were a large number e.g. 1000 then SQL would generate an error. You can set the lmit to 0 which means unbounded, but only do this after testing your code, because it can get stuck in an infinite loop this way (which is why the limit exists by default).

How to use regular expression to remove number in MS SQL Server Management Studio?

I have a field in a table containing different IDs for different programmes like this:
ProgrammeID
-----------
Prog201604L
Prog201503L
Pro2015N
Pro2014N
Programme2010
Programme2011
Each programme ID has its meaning. The number in the mid of the string indicates the time or month. It is obvious that Prog201604L and Prog201503L indicate the same programme but in different years (so do the rest). What I want to do is to remove the numbers so after removal the programmeID will be like:
ProgrammeID
-----------
ProgL
ProgL
ProN
ProN
Programme
Programme
Then later I can aggregate this programmes together.
I am currently using SSMS 2012 not sure if there is a sql statement like RegEx. I have been searching for a long time but the solution online are mainly about Oracle and MySQL. What I found is PATINDEX() and it seems to support regular expression. Can anybody tell me how to create a pattern that suits my situation and what kind of statement I should use?
Thanks in advance
If the Number part is always 6 characters below can be used.
DECLARE #ProgrammeID VARCHAR(50) = 'Prog201604L'
SELECT STUFF(#ProgrammeID, PATINDEX( '%[0-9]%', #ProgrammeID), 6, '')
If the numbers are not fixed... to extend above
CREATE TABLE #Programme ( ProgrammeID VARCHAR(50) )
INSERT INTO #Programme
VALUES
('Prog201604L')
,('Pro2015N')
,('Programme2010')
,('Prog2016L')
,('Pro2N')
,('Prog')
,('2010')
SELECT ProgrammeID,
ISNULL(
STUFF(ProgrammeID,
PATINDEX( '%[0-9]%', ProgrammeID), -- get number start index
IIF(PATINDEX( '%[0-9][a-z]%',ProgrammeID)= 0, PATINDEX( '%[0-9]',ProgrammeID), PATINDEX( '%[0-9][a-z]%',ProgrammeID)) + 1 -- get the last number index
- PATINDEX( '%[0-9]%', ProgrammeID), -- get the number character length
'')
,ProgrammeID) -- Where there are no numbers in the string you will get Null, replace it with actual string
AS [Without Numbers]
FROM #Programme
this will handle cases with varying numbers and even string without number.
Hope this helps
You can create a function and pass the value of each row to function
as (just run this query)
Create Function [dbo].[RemoveNonAlphaCharacters](#Temp VarChar(1000))
Returns VarChar(1000)
AS
Begin
Declare #KeepValues as varchar(50)
Set #KeepValues = '%[^a-z]%'
While PatIndex(#KeepValues, #Temp) > 0
Set #Temp = Stuff(#Temp, PatIndex(#KeepValues, #Temp), 1, '')
Return #Temp
End
---Call it like this:
Declare #tbl table (ProgrammeID varchar(20))
insert into #tbl values ('ProgL'),('ProgL'),('ProN'),('ProN'),('Programme'),('Programme')
select * from #tbl
Select dbo.RemoveNonAlphaCharacters(ProgrammeID) from #tbl
How to strip all non-alphabetic characters from string in SQL Server?
Remove numbers from string sql server
One clever option is to take the substring of the ProgrammeID column from the left, until hitting the first number, and concatenate that with the reverse of the substring from the right until hitting the first number:
SELECT
SUBSTRING(ProgrammeID,
1,
PATINDEX('%[0-9]%', ProgrammeID) - 1) +
REVERSE(SUBSTRING(REVERSE(ProgrammeID),
1,
PATINDEX('%[0-9]%', REVERSE(ProgrammeID)) - 1))
FROM yourTable
I have created a user-defined function for SQL Server to remove non-numeric characters in a string expression
We can modify it to remove the opposite, numeric characters from the input string as follows
while patindex('%[0-9]%', #str) > 0
set #str = stuff(#str, patindex('%[0-9]%', #str), 1, '')
return #str
I hope it helps
Alan Burstein wrote an iTVF exactly for this. The function is called PatExclude8K. Here is the function definition (some comments removed):
CREATE FUNCTION dbo.PatExclude8K
(
#String VARCHAR(8000),
#Pattern VARCHAR(50)
)
/*******************************************************************************
Purpose:
Given a string (#String) and a pattern (#Pattern) of characters to remove,
remove the patterned characters from the string.
*******************************************************************************/
RETURNS TABLE WITH SCHEMABINDING AS
RETURN
WITH
E1(N) AS (SELECT N FROM (VALUES (NULL),(NULL),(NULL),(NULL),(NULL),(NULL),(NULL),(NULL),(NULL),(NULL)) AS X(N)),
itally(N) AS
(
SELECT TOP(CONVERT(INT,LEN(#String),0)) ROW_NUMBER() OVER (ORDER BY (SELECT NULL))
FROM E1 T1 CROSS JOIN E1 T2 CROSS JOIN E1 T3 CROSS JOIN E1 T4
)
SELECT NewString =
((
SELECT SUBSTRING(#String,N,1)
FROM iTally
WHERE 0 = PATINDEX(#Pattern,SUBSTRING(#String COLLATE Latin1_General_BIN,N,1))
FOR XML PATH(''),TYPE
).value('.[1]','varchar(8000)'));
GO
And here is how you would use it:
SELECT *
FROM #Programme p
CROSS APPLY dbo.PatExclude8K(p.ProgrammeID, '[0-9]');
Using your sample data, here is the result:
ProgrammeID NewString
-------------------- -----------------
Prog201604L ProgL
Prog201503L ProgL
Pro2015N ProN
Pro2014N ProN
Programme2010 Programme
Programme2011 Programme
I created this solution building on a solution to extracting values from a comma separated list inside a string.
It seems to work find and even be a bit more effective than using while - I will be happy for feedback about that assumption, though.
On on table with 461.358 rows it takes 3 minutes and 27 seconds to do this (0.44 ms per row) (I put it into a function).
select count(*)
from Mytable
where dbo.StripNumeric(inputFromUser) is null
Here's the solutions
For stripping away numeric:
declare #input nvarchar(max) = null
select #input = '1a2 3b4' + char(13) + char(10) + '5(678)*90c'
DECLARE #output nvarchar(max) = '';
WITH cte AS
(
SELECT cast(1 as int) as [index]
UNION ALL
SELECT [index]+ 1 as [index]
from cte
where [index] < len(#input)
)
select #output = iif(PATINDEX('%[0-9]%', substring(#input, [index], 1))= 1, #output, #output + substring(#input, [index], 1))
from cte;
select iif(COALESCE( #output, '') = '', null, ltrim(rtrim(#output)))
For stripping away non-numeric:
declare #input nvarchar(max) = null
select #input = '1a2 3b4' + char(13) + char(10) + '5(678)*90c'
DECLARE #output nvarchar(max) = '';
WITH cte AS
(
SELECT cast(1 as int) as [index]
UNION ALL
SELECT [index]+ 1 as [index]
from cte
where [index] < len(#input) --len(substring(#input, index, 1)) >
)
select #output = iif(PATINDEX('%[0-9]%', substring(#input, [index], 1))= 1, #output + substring(#input, [index], 1), #output)
from cte;
select iif(COALESCE( #output, '') = '', null, ltrim(rtrim(#output)))

SQL: iterating through a list that contains some ranges

I am trying to get information for products that have an ID that is contained in a list. The problem is that the list contains some single values and some range values:
PX03 - PX069, PX20, PX202, PX25 - PX270, PX250 - PX2509, PX251, PX2511 -
PX2513
Basically what I am looking for is some way to take a list or string containing both values and ranges and the end output is a table or list that has all of the values within the ranges individually so that I can loop through them.
I have a stored procedure that loops through all the ID's in the main products table that use the 'PX' prefix, but the table has all ids (i.e. PX 1 - 9999, LX 00001 - 99999) and I only want to search through those contained in the above list. I could write it out all the id's individually but some of the ranges contain many values, which would be time consuming to go through.
My idea was to create a separate table containing this list, in which there would be three columns: an identity column, and then one column each for the beginning and end of the range. Any items that do not have a range would just have the same value for beginning and end range, i.e.:
----------------------------------
rownum | range_start | range_end|
----------------------------------
1 PX03 PX069
2 PX20 PX20
3 PX202 PX202
4 PX25 PX25
5 PX250 PX2509
and then populating a table using something like:
SELECT id from product_table
WHERE id BETWEEN listtable.range_start AND listtable.range_end
where product_table is my original table with the product id's and their information and listtable is the new table I just created. This would give me:
id|
---
PX03
PX030
PX031
PX032
PX033
.
.
.
PX067
PX068
PX069
PX20
PX202
PX25
PX250
PX251
etc.
but I am thinking I would need to iterate through the list and I am not sure how to do that. Any ideas, hints or suggestions?
UPDATE
After creating the table using the solution given by #asantaballa, it was as simple as using an inner join:
SELECT d.id
FROM product_table d
INNER JOIN #RangeTable r
ON d.id BETWEEN r.RangeFrom AND r.RangeTo
See if this works for you for the part about converting the string to a table.
Declare #StrList Varchar(1000) = 'PX03 - PX069, PX20, PX202, PX25 - PX270, PX250 - PX2509, PX251, PX2511 - PX2513'
Declare #RangeTable Table (RangeFrom VarChar(32), RangeTo VarChar(32))
Select #StrList = Replace(#StrList,' ', '') + ','
Declare #StrListItem Varchar(32)
While CHARINDEX(',', #StrList) > 0
Begin
Select #StrListItem = SUBSTRING(#StrList,1,CHARINDEX(',', #StrList) - 1)
Declare
#RangeFrom VarChar(32)
, #RangeTo VarChar(32)
If CHARINDEX('-', #StrListItem) = 0
Begin
Select
#RangeFrom = #StrListItem
, #RangeTo = #StrListItem
End
Else
Begin
Select
#RangeFrom = SUBSTRING(#StrListItem, 1, CHARINDEX('-', #StrListItem) - 1)
, #RangeTo = SUBSTRING(#StrListItem, CHARINDEX('-', #StrListItem) + 1, LEN(#StrListItem) - CHARINDEX('-', #StrListItem))
End
Insert Into #RangeTable (RangeFrom, RangeTo) Values (#RangeFrom, #RangeTo)
Select #StrList = SUBSTRING(#StrList, CHARINDEX(',', #StrList) + 1, LEN(#StrList) - CHARINDEX(',', #StrList))
End
Select * From #RangeTable
Here is your string and product_table
DECLARE #STR VARCHAR(100) = 'PX03 - PX069, PX20, PX202, PX25 - PX270, PX250 - PX2509, PX251, PX2511 - PX2513'
SELECT * INTO #product_table
FROM
(
SELECT 'PX4' PRODID
UNION ALL
SELECT 'PX26'
UNION ALL
SELECT 'PX75'
UNION ALL
SELECT 'PX77'
)TAB
Now create a table to hold the value
CREATE TABLE #listtable(ROWNUM int IDENTITY(1,1),range_start VARCHAR(100),range_end VARCHAR(100))
Now insert the splitted value to the table.
INSERT INTO #listtable
SELECT
ISNULL(PARSENAME(REPLACE(Split.a.value('.', 'VARCHAR(100)'),'-','.'),2),Split.a.value('.', 'VARCHAR(100)')) 'range_start' ,
PARSENAME(REPLACE(Split.a.value('.', 'VARCHAR(100)'),'-','.'),1) 'range_end'
FROM
(
SELECT CAST ('<M>' + REPLACE(#STR, ',', '</M><M>') + '</M>' AS XML) AS Data
) AS A
CROSS APPLY Data.nodes ('/M') AS Split(a)
Since Id is string, you need a function to extract numbers from Id(function created by God of SQL Server - Pinal Dave)
CREATE FUNCTION dbo.udf_GetNumeric
(#strAlphaNumeric VARCHAR(256))
RETURNS VARCHAR(256)
AS
BEGIN
DECLARE #intAlpha INT
SET #intAlpha = PATINDEX('%[^0-9]%', #strAlphaNumeric)
BEGIN
WHILE #intAlpha > 0
BEGIN
SET #strAlphaNumeric = STUFF(#strAlphaNumeric, #intAlpha, 1, '' )
SET #intAlpha = PATINDEX('%[^0-9]%', #strAlphaNumeric )
END
END
RETURN ISNULL(#strAlphaNumeric,0)
END
First of all keep in mind that we will not get PX1,PX2,PX3,PX4 if you give id BETWEEN listtable.range_start AND listtable.range_end because those are of varchar type and not numbers. So we need to extract numbers from each PX and get the values between them and append PX.
Here is the query which filters the IDs in product_table which are in the range between listtable
;WITH CTE AS
(
SELECT ROWNUM,CAST(dbo.udf_GetNumeric(range_start)AS INT) NUMBERS,
CAST(dbo.udf_GetNumeric(range_end)AS INT) RTO1
FROM #listtable
UNION ALL
SELECT T.ROWNUM,NUMBERS+1,RTO1
FROM #listtable T
JOIN CTE ON CTE.ROWNUM = T.ROWNUM
WHERE NUMBERS < RTO1
)
SELECT PRODID IDS--,ROWNUM,NUMBERS NUMS,'PX'+CAST(NUMBERS AS VARCHAR(10)) IDS2
FROM CTE
JOIN #product_table ON PRODID='PX'+CAST(NUMBERS AS VARCHAR(10))
ORDER BY NUMBERS
option (MaxRecursion 0)
SQL FIDDLE

Resources