I'm trying to create a stored procedure to import from CSV. Everything works if I have a hard coded file path, but I want to take a file path as a parameter. When I try SQL Sever Management Studio generates an error:
Incorrect syntax near '#filePath'.
(In fact, if I put anything but a pure string(eg. 'C:'+'/dir') it gives an error.)
This is a simplified version of my code:
Create procedure [importFile](#filePath varchar(Max))
AS
BEGIN
create table #Temp
(
row1 int,
row2 varchar(5),
row3 bit
)
BULK insert
#Temp
from #filePath
With(
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
...
END
Any explanation?
Use dynamic SQL to inject the file name variable into a string with the bulk insert statement and the use sp_executesqlto execute it. You might want to add some error checking to check that the path is valid and so on.
CREATE PROCEDURE [importFile] (#filePath VARCHAR(MAX))
AS
BEGIN
CREATE TABLE #Temp
(
row1 int,
row2 varchar(5),
row3 bit
)
DECLARE #SQL NVARCHAR(MAX) = ''
SET #SQL = N'
BULK INSERT #Temp
FROM ''' + #filePath + '''
WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n''
)'
-- ...
EXEC sp_executesql #SQL
END
-- to run it:
EXEC importFile 'd:\test.csv'
Related
I am trying to insert data from multiple CSV files from a single folder to a single table.
I can do bulinsert for 1 file using the following code:
USE [dbname]
GO
BULK INSERT tablename
FROM 'path to csv files'
WITH
(
FIRSTROW = 2, -- as 1st one is header
FIELDTERMINATOR = ',', --CSV field delimiter
ROWTERMINATOR = '\n', --Use to shift the control to next row
TABLOCK
)
GO
The first thing I would say is I personally wouldn't do this in T-SQL, I'd use something a bit better suited to the task, the SSIS Multiple flat file connection manager for example. This is possible in T-SQL but it is not the most robust approach and has limitation (such as the directories that can even be accessed).
-- Set the directory to check for files
DECLARE #Directory NVARCHAR(200) = N'C:\Your File Location\';
-- Create a table variable to store the files output from xp_dirtree
DECLARE #Files TABLE (FileName NVARCHAR(255), Depth INT, IsFile BIT);
-- Get the files from the folder
INSERT #Files(FileName, Depth, IsFile)
EXECUTE master..xp_dirtree #Directory, 1, 1;
--select the filenames and use STRING_AGG to combine into a single string to execute
DECLARE #SQL NVARCHAR(MAX) =
( SELECT STRING_AGG(CONCAT('BULK INSERT YourTableName FROM ''',
CONCAT(#Directory, f.FileName), '''
WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 1
)'), ';' + CHAR(10))
FROM #Files AS f
WHERE f.IsFile = 1
AND f.FileName LIKE '%.csv'-- Optionally limit files
);
PRINT #SQL;
-- Uncomment below line once you're happy with the SQL Printed
--EXECUTE sp_executesql #SQL;
Or if you want a bit more control over error handling, you could use a cursor to iterate the files:
-- Set the directory to check for files
DECLARE #Directory NVARCHAR(200) = N'C:\Import Location\';
-- Create a table variable to store the files output from xp_dirtree
DECLARE #Files TABLE (FileName NVARCHAR(255), Depth INT, IsFile BIT);
-- Get the files from the folder
INSERT #Files(FileName, Depth, IsFile)
EXECUTE master..xp_dirtree #Directory, 1, 1;
--declare a cursor to loop through the files, filtered for csv
DECLARE FileCursor CURSOR LOCAL STATIC FORWARD_ONLY READ_ONLY
FOR
SELECT FilePath = CONCAT(#Directory, f.FileName)
FROM #Files AS f
WHERE f.IsFile = 1
AND f.FileName LIKE '%.csv'; -- Optionally limit files
OPEN FileCursor;
DECLARE #FilePath NVARCHAR(255);
FETCH FROM FileCursor INTO #FilePath;
WHILE ##FETCH_STATUS = 0
BEGIN
PRINT #FilePath;
DECLARE #sql NVARCHAR(MAX) =
CONCAT('BULK INSERT YourTableName FROM ''', #FilePath, '''
WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 1
)');
BEGIN TRY
EXECUTE sp_executesql #SQL;
END TRY
BEGIN CATCH
-- Do something to handle errors
END CATCH
FETCH NEXT FROM FileCursor INTO #FilePath;
END
CLOSE FileCursor;
DEALLOCATE FileCursor;
As I say though, SSIS or another dedicated ETL tool is a better choice.
DECLARE #SQL_BULK VARCHAR(MAX)
SET #SQL_BULK = 'BULK INSERT [dbo].[Table]
FROM ''' + #BatchFileName + '''
WITH
(
FIRSTROW = 2,
FIELDTERMINATOR = ''\t'',
ROWTERMINATOR = ''0x0a''
)'
EXEC (#SQL_BULK)
I have this code to do a bulk load. Works fine, but I would like to have the #BatchFileName also in there as a column (each row contains the same value).
Is this possible during the bulk load? Or how can I add it later on in a separate function?
Thanks in advance
i use this script to iterate and bulkinsert files in directory:
create table #x (name varchar(200))
DECLARE #query varchar (1000),#conta int ,#query2 varchar (1000),#NOME varchar(50)
set #conta=1
set #query ='master.dbo.xp_cmdshell "dir '+'C:\directoryname'+'*.csv' +' /b"'
insert #x exec (#query)
delete from #x where name is NULL
select identity(int,1,1) as ID, name into #y from #x
drop table #x
WHILE #conta<221 --number of files
BEGIN
SELECT #NOME=name FROM #y WHERE ID=#conta
set #Query2 ='BULK INSERT [dbo].[tablename] FROM '+ '''C:\directoryname'+#NOME+'''
WITH (
FIELDTERMINATOR = '','',ROWTERMINATOR = ''0x0a'')'
SELECT #query2
exec (#Query2)
set #conta=#conta+1
END
drop table #y
I have the following code to create a SQL function that will parse an XML string and create a table of key value pairs that represent the nodes and values. This works fine for me in my use cases.
CREATE FUNCTION XmlToKeyValue
(
#rootName AS varchar(256),
#xml AS Xml
)
RETURNS #keyval TABLE ([key] varchar(max), [value] varchar(max))
AS
BEGIN
DECLARE #input TABLE (XmlData XML NOT NULL)
INSERT INTO #input VALUES(#xml)
INSERT #keyval ([key], [value])
SELECT
XC.value('local-name(.)', 'varchar(max)') AS [key],
XC.value('(.)[1]', 'varchar(max)') AS [value]
FROM
#input
CROSS APPLY
XmlData.nodes('/*[local-name()=sql:variable("#rootName")]/*') AS XT(XC)
RETURN
END
What I am trying to do is have a stored procedure in my main database that will create another database with all the appropriate functions/procedures/etc. So in that stored procedure I am trying to do something like this:
SET #cmd = '
CREATE FUNCTION XmlToKeyValue
(
#rootName AS varchar(256),
#xml AS Xml
)
RETURNS #keyval TABLE ([key] varchar(max), [value] varchar(max))
AS
BEGIN
DECLARE #input TABLE (XmlData XML NOT NULL)
INSERT INTO #input VALUES(#xml)
INSERT #keyval ([key], [value])
SELECT
XC.value(''local-name(.)'', ''varchar(max)'') AS [key],
XC.value(''(.)[1]'', ''varchar(max)'') AS [value]
FROM
#input
CROSS APPLY
XmlData.nodes(''/*[local-name()=sql:variable("#rootName")]/*'') AS XT(XC)
RETURN
END
'
BEGIN TRY
EXEC(N'USE '+#dbName+'; EXEC sp_executesql N''' + #cmd + '''; USE master')
END TRY
BEGIN CATCH
PRINT 'Error creating XmlToKeyValue'
Print Error_Message();
RETURN
END CATCH
However, I am getting the following error that I can't figure out how to resolve.
Error creating XmlToKeyValue
Incorrect syntax near 'local'.
Can I use local-name in a dynamic sql statement? If not, how can I achieve my goal? Thank you.
The problem is not the local-name function. It is entirely the fact that you are concatenating in the #cmd variable into your Dynamic SQL without properly escaping the embedded single-quotes.
This line:
EXEC(N'USE '+#dbName+'; EXEC sp_executesql N''' + #cmd + '''; USE master')
should be:
SET #cmd = REPLACE(#cmd, N'''', N'''''');
EXEC(N'USE ' + #dbName + N'; EXEC sp_executesql N''' + #cmd + N''';');
Else you are embedding:
XC.value(''local-name(
into the string, but using the same number of escape sequences, hence the XC.value( now becomes the end of the string and the local-name(.) is technically unescaped SQL and not part of a string.
Also:
You don't need the USE master at the end of the Dynamic SQL (so I removed it).
You prefixed the first string literal with N but none of the others (I added the N in for the others so that they all have that prefix).
I'm still getting used to SQL, so before I get to using stored procedure, I would like to understand how to use BULK INSERT effectively first.
I need to combine 50+ csv files and dump them into an SQL table. The problem is, I'd like to be able to tell each record apart (as in, each record belongs to a certain csv file, which I will identify by the file name).
Here's a small example of what I want:
CREATE TABLE ResultsDump
(
PC FLOAT,
Amp VARCHAR(50),
RCS VARCHAR(50),
CW VARCHAR(50),
State0 VARCHAR(50),
State1 VARCHAR(50),
)
BULK INSERT ResultsDump
FROM 'c:\distance1000_7_13_2010_1_13PM_Avery DennisonAD_2300008_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
BULK INSERT ResultsDump
FROM 'c:\distance1000_7_13_2010_2_27PM_Avery DennisonAD_2300009_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
BULK INSERT ResultsDump
FROM 'C:\distance1000_7_13_2010_2_58PM_Avery DennisonAD_230000A_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
BULK INSERT ResultsDump
FROM 'c:\distance1000_7_13_2010_3_21PM_Avery DennisonAD_230000B_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
BULK INSERT ResultsDump
FROM 'c:\distance1000_7_13_2010_3_41PM_Avery DennisonAD_230000C_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
I know this is an inefficient way of doing things, but I definitely like to figure out how to manually dump one file in the SQL table in the format I want before I start to create a stored procedure.
In the new table, I want something like this:
FileName,PC,Amp,RCS,CW,State0,State1
c:\distance1000_7_13_2010_1_13PM_Avery DennisonAD_2300008_10S_Lock.csv, ...
...
...
c:\distance1000_7_13_2010_2_27PM_Avery DennisonAD_2300009_10S_Lock.csv, ...
...
...
c:\distance1000_7_13_2010_2_58PM_Avery DennisonAD_230000A_10S_Lock.csv, ...
...
...
Any simple suggestions or referrals to specific functions would be great! Remember, I'm getting used to SQL and it'd be great if I could take this one step at a time, that's why I'm starting with such a simple question.
Thanks in advance!
You can add a column FileName varchar(max) to the ResultsDump table, create a view of the table with the new column, bulk insert into the view, and after every insert, set the filename for columns where it still has its default value null:
CREATE TABLE dbo.ResultsDump
(
PC FLOAT,
Amp VARCHAR(50),
RCS VARCHAR(50),
CW VARCHAR(50),
State0 VARCHAR(50),
State1 VARCHAR(50),
)
GO
ALTER TABLE dbo.ResultsDump ADD [FileName] VARCHAR(300) NULL
GO
CREATE VIEW dbo.vw_ResultsDump AS
SELECT
PC,
Amp,
RCS,
CW,
State0,
State1
FROM
ResultsDump
GO
BULK INSERT vw_ResultsDump
FROM 'c:\distance1000_7_13_2010_1_13PM_Avery DennisonAD_2300008_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
UPDATE dbo.ResultsDump
SET [FileName] = 'c:\distance1000_7_13_2010_1_13PM_Avery DennisonAD_2300008_10S_Lock.csv'
WHERE [FileName] IS NULL
BULK INSERT vw_ResultsDump
FROM 'c:\distance1000_7_13_2010_2_27PM_Avery DennisonAD_2300009_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
UPDATE dbo.ResultsDump
SET [FileName] = 'distance1000_7_13_2010_2_27PM_Avery DennisonAD_2300009_10S_Lock.csv'
WHERE [FileName] IS NULL
Try this,
ALTER PROCEDURE [dbo].[ReadandUpdateFileNames_SP]
(
#spRequestId NVARCHAR(50)
,#LoopCounter INT =0
,#MaxFIVId INT=0
,#spFileName NVARCHAR(100)=NULL)
AS
BEGIN
SET NOCOUNT ON
BEGIN TRY
BEGIN TRAN
-- To read filename's from the Request Id and store it in a temp table
DECLARE #cmd nvarchar(500)
SET #cmd = 'dir D:\Input\REQUEST-'+#spRequestId+' /b '
--PRINT #cmd
DECLARE #DirOutput TABLE(
ID INT IDENTITY
, files varchar(500))
INSERT INTO #DirOutput
EXEC master.dbo.xp_cmdshell #cmd
SELECT * FROM #DirOutput WHERE files IS NOT NULL ORDER BY ID
----Read files by RequestId BEGIN
SELECT #LoopCounter = min(ID) , #MaxFIVId = max(ID)
FROM #DirOutput
WHILE(#LoopCounter IS NOT NULL AND #LoopCounter<#MaxFIVId)
BEGIN
-- Create temp table to store FIVItems
CREATE TABLE Items_TEMP
(
ControlID NVARCHAR(50)
, UNRS_Code NUMERIC(18,0)
, UNRS_Code_S NUMERIC(18,0)
, Ordered_Quantity NUMERIC(18,3)
, Sent_Quantity NUMERIC(18,3)
, Accepted_Quantity NUMERIC(18,3)
, Unit_Food_Price NUMERIC(18,2)
, Total_Price NUMERIC(18,2)
)
SELECT #spFileName=files FROM #DirOutput WHERE ID=#LoopCounter
PRINT #LoopCounter
DECLARE #spControlId NVARCHAR(50)
SET #spControlId='FFO'+ Substring(#spFileName, 4, (len(#spFileName)-7))
--PRINT #spControlId
DECLARE #sqlCmd NVARCHAR(MAX)
SET #sqlCmd='BULK INSERT
Items_TEMP
FROM ''D:\Input\REQUEST-'+#spRequestId+'\'+#spFileName+'''
WITH(
FIELDTERMINATOR='',''
, ROWTERMINATOR=''\n''
)'
PRINT #sqlCmd
EXECUTE sp_executesql #sqlCmd
---Add a new column to the table which is not present in the CSV
ALTER TABLE Items_TEMP ADD OrderId NUMERIC(18,0)NULL
UPDATE Items_TEMP SET ControlID=#spControlId,OrderId=(SELECT OrderId FROM dbo.Orders WHERE ControlID=#spControlId)
SELECT * FROM Items_TEMP
--DROP FIVItems_TEMP table once CSV output generated
DROP TABLE Items_TEMP
SET #LoopCounter=#LoopCounter+1
END
----****END***
COMMIT TRAN
END TRY
BEGIN CATCH
ROLLBACK TRAN
PRINT 'ROLLBACK'
PRINT Error_Message()
SELECT ERROR_LINE() AS ErrorLine;
END CATCH
SET NOCOUNT OFF
END
The code is as follows:
ALTER PROCEDURE dbo.pdpd_DynamicCall
#SQLString varchar(4096) = null
AS
Begin
create TABLE #T1 ( column_1 varchar(10) , column_2 varchar(100) )
insert into #T1
execute ('execute ' + #SQLString )
select * from #T1
End
The problem is that I want to call different procedures that can give back different columns.
Therefore I would have to define the table #T1 generically.
But I don't know how.
Can anyone help me on this problem?
Try:
SELECT into #T1 execute ('execute ' + #SQLString )
And this smells real bad like an sql injection vulnerability.
correction (per #CarpeDiem's comment):
INSERT into #T1 execute ('execute ' + #SQLString )
also, omit the 'execute' if the sql string is something other than a procedure
You can define a table dynamically just as you are inserting into it dynamically, but the problem is with the scope of temp tables. For example, this code:
DECLARE #sql varchar(max)
SET #sql = 'CREATE TABLE #T1 (Col1 varchar(20))'
EXEC(#sql)
INSERT INTO #T1 (Col1) VALUES ('This will not work.')
SELECT * FROM #T1
will return with the error "Invalid object name '#T1'." This is because the temp table #T1 is created at a "lower level" than the block of executing code. In order to fix, use a global temp table:
DECLARE #sql varchar(max)
SET #sql = 'CREATE TABLE ##T1 (Col1 varchar(20))'
EXEC(#sql)
INSERT INTO ##T1 (Col1) VALUES ('This will work.')
SELECT * FROM ##T1
Hope this helps,
Jesse
Be careful of a global temp table solution as this may fail if two users use the same routine at the same time as a global temp table can be seen by all users...
create a global temp table with a GUID in the name dynamically. Then you can work with it in your code, via dyn sql, without worry that another process calling same sproc will use it. This is useful when you dont know what to expect from the underlying selected table each time it runs so you cannot created a temp table explicitly beforehand. ie - you need to use SELECT * INTO syntax
DECLARE #TmpGlobalTable varchar(255) = 'SomeText_' + convert(varchar(36),NEWID())
-- select #TmpGlobalTable
-- build query
SET #Sql =
'SELECT * INTO [##' + #TmpGlobalTable + '] FROM SomeTable'
EXEC (#Sql)
EXEC ('SELECT * FROM [##' + #TmpGlobalTable + '] ')
EXEC ('DROP TABLE [##' + #TmpGlobalTable + ']')
PRINT 'Dropped Table ' + #TmpGlobalTable
INSERT INTO #TempTable
EXEC(#SelectStatement)
Try Below code for creating temp table dynamically from Stored Procedure Output using T-SQL
declare #ExecutionName varchar(1000) = 'exec [spname] param1,param2 '
declare #sqlStr varchar(max) = ''
declare #tempTableDef nvarchar(max) =
(
SELECT distinct
STUFF(
(
SELECT ','+a.[name]+' '+[system_type_name]
+'
' AS [text()]
FROM sys.dm_exec_describe_first_result_set (#ExecutionName, null, 0) a
ORDER BY a.column_ordinal
FOR XML PATH ('')
), 1, 1, '') tempTableDef
FROM sys.dm_exec_describe_first_result_set (#ExecutionName, null, 0) b
)
IF ISNULL(#tempTableDef ,'') = '' RAISERROR( 'Invalid SP Configuration. At least one column is required in Select list of SP output.',16,1) ;
set #tempTableDef='CREATE TABLE #ResultDef
(
' + REPLACE(#tempTableDef,'
','') +'
)
INSERT INTO #ResultDef
' + #ExecutionName
Select #sqlStr = #tempTableDef +' Select * from #ResultDef '
exec(#sqlStr)
DECLARE #EmpGroup INT =3 ,
#IsActive BIT=1
DECLARE #tblEmpMaster AS TABLE
(EmpCode VARCHAR(20),EmpName VARCHAR(50),EmpAddress VARCHAR(500))
INSERT INTO #tblEmpMaster EXECUTE SPGetEmpList #EmpGroup,#IsActive
SELECT * FROM #tblEmpMaster
CREATE PROCEDURE dbo.pdpd_DynamicCall
AS
DECLARE #SQLString_2 NVARCHAR(4000)
SET NOCOUNT ON
Begin
--- Create global temp table
CREATE TABLE ##T1 ( column_1 varchar(10) , column_2 varchar(100) )
SELECT #SQLString_2 = 'INSERT INTO ##T1( column_1, column_2) SELECT column_1 = "123", column_2 = "MUHAMMAD IMRON"'
SELECT #SQLString_2 = REPLACE(#SQLString_2, '"', '''')
EXEC SP_EXECUTESQL #SQLString_2
--- Test Display records
SELECT * FROM ##T1
--- Drop global temp table
IF OBJECT_ID('tempdb..##T1','u') IS NOT NULL
DROP TABLE ##T1
End
Not sure if I understand well, but maybe you could form the CREATE statement inside a string, then execute that String? That way you could add as many columns as you want.