How do I BULK INSERT with additional column showing filename? - sql-server

Question

I'm still getting used to SQL, so before I get to using stored procedure, I would like to understand how to use BULK INSERT effectively first.
I need to combine 50+ csv files and dump them into an SQL table. The problem is, I'd like to be able to tell each record apart (as in, each record belongs to a certain csv file, which I will identify by the file name).
Here's a small example of what I want:
CREATE TABLE ResultsDump
(
PC FLOAT,
Amp VARCHAR(50),
RCS VARCHAR(50),
CW VARCHAR(50),
State0 VARCHAR(50),
State1 VARCHAR(50),
)
BULK INSERT ResultsDump
FROM 'c:\distance1000_7_13_2010_1_13PM_Avery DennisonAD_2300008_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
BULK INSERT ResultsDump
FROM 'c:\distance1000_7_13_2010_2_27PM_Avery DennisonAD_2300009_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
BULK INSERT ResultsDump
FROM 'C:\distance1000_7_13_2010_2_58PM_Avery DennisonAD_230000A_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
BULK INSERT ResultsDump
FROM 'c:\distance1000_7_13_2010_3_21PM_Avery DennisonAD_230000B_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
BULK INSERT ResultsDump
FROM 'c:\distance1000_7_13_2010_3_41PM_Avery DennisonAD_230000C_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
I know this is an inefficient way of doing things, but I definitely like to figure out how to manually dump one file in the SQL table in the format I want before I start to create a stored procedure.
In the new table, I want something like this:
FileName,PC,Amp,RCS,CW,State0,State1
c:\distance1000_7_13_2010_1_13PM_Avery DennisonAD_2300008_10S_Lock.csv, ...
...
...
c:\distance1000_7_13_2010_2_27PM_Avery DennisonAD_2300009_10S_Lock.csv, ...
...
...
c:\distance1000_7_13_2010_2_58PM_Avery DennisonAD_230000A_10S_Lock.csv, ...
...
...
Any simple suggestions or referrals to specific functions would be great! Remember, I'm getting used to SQL and it'd be great if I could take this one step at a time, that's why I'm starting with such a simple question.
Thanks in advance!

Answer 1

You can add a column FileName varchar(max) to the ResultsDump table, create a view of the table with the new column, bulk insert into the view, and after every insert, set the filename for columns where it still has its default value null:
CREATE TABLE dbo.ResultsDump
(
PC FLOAT,
Amp VARCHAR(50),
RCS VARCHAR(50),
CW VARCHAR(50),
State0 VARCHAR(50),
State1 VARCHAR(50),
)
GO
ALTER TABLE dbo.ResultsDump ADD [FileName] VARCHAR(300) NULL
GO
CREATE VIEW dbo.vw_ResultsDump AS
SELECT
PC,
Amp,
RCS,
CW,
State0,
State1
FROM
ResultsDump
GO
BULK INSERT vw_ResultsDump
FROM 'c:\distance1000_7_13_2010_1_13PM_Avery DennisonAD_2300008_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
UPDATE dbo.ResultsDump
SET [FileName] = 'c:\distance1000_7_13_2010_1_13PM_Avery DennisonAD_2300008_10S_Lock.csv'
WHERE [FileName] IS NULL
BULK INSERT vw_ResultsDump
FROM 'c:\distance1000_7_13_2010_2_27PM_Avery DennisonAD_2300009_10S_Lock.csv'
WITH
(
FIRSTROW = 2,
MAXERRORS = 0,
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
UPDATE dbo.ResultsDump
SET [FileName] = 'distance1000_7_13_2010_2_27PM_Avery DennisonAD_2300009_10S_Lock.csv'
WHERE [FileName] IS NULL

Answer 2

Try this,
ALTER PROCEDURE [dbo].[ReadandUpdateFileNames_SP]
(
#spRequestId NVARCHAR(50)
,#LoopCounter INT =0
,#MaxFIVId INT=0
,#spFileName NVARCHAR(100)=NULL)
AS
BEGIN
SET NOCOUNT ON
BEGIN TRY
BEGIN TRAN
-- To read filename's from the Request Id and store it in a temp table
DECLARE #cmd nvarchar(500)
SET #cmd = 'dir D:\Input\REQUEST-'+#spRequestId+' /b '
--PRINT #cmd
DECLARE #DirOutput TABLE(
ID INT IDENTITY
, files varchar(500))
INSERT INTO #DirOutput
EXEC master.dbo.xp_cmdshell #cmd
SELECT * FROM #DirOutput WHERE files IS NOT NULL ORDER BY ID
----Read files by RequestId BEGIN
SELECT #LoopCounter = min(ID) , #MaxFIVId = max(ID)
FROM #DirOutput
WHILE(#LoopCounter IS NOT NULL AND #LoopCounter<#MaxFIVId)
BEGIN
-- Create temp table to store FIVItems
CREATE TABLE Items_TEMP
(
ControlID NVARCHAR(50)
, UNRS_Code NUMERIC(18,0)
, UNRS_Code_S NUMERIC(18,0)
, Ordered_Quantity NUMERIC(18,3)
, Sent_Quantity NUMERIC(18,3)
, Accepted_Quantity NUMERIC(18,3)
, Unit_Food_Price NUMERIC(18,2)
, Total_Price NUMERIC(18,2)
)
SELECT #spFileName=files FROM #DirOutput WHERE ID=#LoopCounter
PRINT #LoopCounter
DECLARE #spControlId NVARCHAR(50)
SET #spControlId='FFO'+ Substring(#spFileName, 4, (len(#spFileName)-7))
--PRINT #spControlId
DECLARE #sqlCmd NVARCHAR(MAX)
SET #sqlCmd='BULK INSERT
Items_TEMP
FROM ''D:\Input\REQUEST-'+#spRequestId+'\'+#spFileName+'''
WITH(
FIELDTERMINATOR='',''
, ROWTERMINATOR=''\n''
)'
PRINT #sqlCmd
EXECUTE sp_executesql #sqlCmd
---Add a new column to the table which is not present in the CSV
ALTER TABLE Items_TEMP ADD OrderId NUMERIC(18,0)NULL
UPDATE Items_TEMP SET ControlID=#spControlId,OrderId=(SELECT OrderId FROM dbo.Orders WHERE ControlID=#spControlId)
SELECT * FROM Items_TEMP
--DROP FIVItems_TEMP table once CSV output generated
DROP TABLE Items_TEMP
SET #LoopCounter=#LoopCounter+1
END
----****END***
COMMIT TRAN
END TRY
BEGIN CATCH
ROLLBACK TRAN
PRINT 'ROLLBACK'
PRINT Error_Message()
SELECT ERROR_LINE() AS ErrorLine;
END CATCH
SET NOCOUNT OFF
END