I am trying to import 24 .txt files to the server but I have limited privileges. All files start with the same characters, but '*' this masking cannot be used for this query. I use below script.
SELECT * into TABLE
FROM OPENROWSET( BULK 'c:\path\files\good*.txt', FORMATFILE = 'c:\path\files\import.xml',FIRSTROW = 2) AS DATA;
You would need to create the table first and then insert each file using a loop.
try the following script after creating the destination table:
declare #filepath varchar(100)= 'c:\path\files\'
,#pattern varchar(100)= 'good*.txt'
,#TableName varchar(100)= 'TestTable'
DECLARE #query varchar(1000)
DECLARE #numfiles int
DECLARE #filename varchar(100)
DECLARE #files TABLE (SourceFileName varchar(200) NULL)
SET #query = 'master.dbo.xp_cmdshell "dir ' + #filepath+#pattern + ' /b"'
INSERT #files(SourceFileName)
EXEC (#query)
DECLARE CUR CURSOR FAST_FORWARD FOR
SELECT SourceFileName FROM #files WHERE SourceFileName IS NOT NULL
SET #numfiles =0
OPEN CUR
FETCH NEXT FROM CUR INTO #filename
WHILE (##FETCH_STATUS = 0)
BEGIN
print #filename
SET #numfiles+=1
SET #query = ('BULK INSERT ' + #TableName
+ ' FROM ''' + #Filepath+#filename + ''' WITH(
FORMATFILE = ''c:\path\files\import.xml'',
FIRSTROW = 2
);'
)
PRINT #query
EXEC (#query)
FETCH NEXT FROM CUR INTO #filename
END
CLOSE CUR
DEALLOCATE CUR
I have a folder called "Data" This folder consists of various .CSV Files with same schema. The folder Location is 'C:\Data..'
I want to Import the contents of all files recursively into SQL Server in one table.
Append additional column in the table which consists for each file.
Have tried some solutions found it here
, but it didn't work.
--get the list of files to process:
SET #path = 'C:\Data\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%'
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql = 'BULK INSERT Temp FROM ''' + #path + #filename + ''' '
+ ' WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
) '
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
TWo things which aren't working as expected:
1) It doesn't work for recursive files in the directory.
2) Not able to do the bulk insert to Temp file created.
I have this SQL command which will be used in a stored procedure and will be scheduled:
BULK INSERT Test101.dbo.Test102
FROM 'C:\Bulk\samp.csv'
WITH
(
FIELDTERMINATOR = ',',
ROWTERMINATOR = '\n'
)
It works well but what I want to do is to process all the .csv files from a folder (let's say Bulk folder) automatically. I mean, the user doesn't have to define the exact location with filename instead the stored procedure will process all the .CSV file from that folder and ignore the other files if there were.
Please help.
Thank you.
try these to make sure you have the right permissions and settings to do xp_cmdshell
One Time config change to enable here : https://stackoverflow.com/a/5131503/2628302
--To Test if that worked run these commands. There should be no errors and should return list of files under c:\
declare #files table (ID int IDENTITY, FileName varchar(500))
insert into #files execute xp_cmdshell 'dir c:\ /b'
select * from #files
--if no errors then proceed to create this SP as shown below. This is the one that does all the work
CREATE PROCEDURE dbo.sp_BulkInsAllFilesInDirectory
AS
BEGIN
--a table to hold filenames
Declare #ALLFILENAMES as TABLE (WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'C:\Bulk\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO #ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE #ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM #ALLFILENAMES where WHICHFILE like '%.csv%'
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql = 'BULK INSERT Temp FROM ''' + #path + #filename + ''' '
+ ' WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
) '
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
END
--- TEST it by running it like so (start with just one csv file in C:\BULK\ directory. If it works for one it will most likely work for more than one file.
EXEC dbo.sp_BulkInsAllFilesInDirectory
see if there are errors. Leave a message here and I will check tomorrow. Good luck.
I'm trying to copy multiple files from specific folder and subfolders into MS SQL 2012. I want to get file details and import them in database as well. I want to get following file details:
Date Created
Date Modified
Size
This is the query that I have:
DECLARE #sql nvarchar(max)
, #filename nvarchar(256)
, #filepath varchar(100)
DECLARE #FileList TABLE (filename nvarchar(256), depth int, fl int)
INSERT INTO #FileList
EXEC xp_dirtree #filepath, 10, 1;
DECLARE curDoc CURSOR FOR
SELECT filename FROM #FileList WHERE fl = 1
OPEN curDoc FETCH NEXT FROM curDoc INTO #filename
WHILE (##FETCH_STATUS = 0)
BEGIN
SET #sql = 'INSERT INTO _table SELECT * FROM OPENROWSET(BULK ''' + #filepath + #filename + ''', SINGLE_BLOB) as tbl'
PRINT #sql
EXEC sp_executesql #sql
FETCH NEXT FROM curDoc
INTO #filename
END
CLOSE curDoc
DEALLOCATE curDoc
I have a folder called "Dump." This folder consists of various .CSV Files.
The folder Location is 'C:\Dump'
I want to Import the contents of these files into SQL Server.
I want the rough code along with proper comments so that I understand it.
I have tried a few codes that I found on the Net. But they haven't quite worked out for me for some strange reason.
The steps I would like to have are
Step 1: Copy all the File Names in the folder to a Table
Step 2: Iterate through the table and copy the data from the files using Bulk Insert.
Someone do please help me out on this one. Thanks a lot in advance :)
--BULK INSERT MULTIPLE FILES From a Folder
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'C:\Dump\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%'
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql = 'BULK INSERT Temp FROM ''' + #path + #filename + ''' '
+ ' WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
) '
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
--Extras
--delete from ALLFILENAMES where WHICHFILE is NULL
--select * from ALLFILENAMES
--drop table ALLFILENAMES
This will give you separate tables for each file.
--BULK INSERT MULTIPLE FILES From a Folder
drop table allfilenames
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'D:\Benihana\backup_csv_benihana_20191128032207_part_1\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
delete from ALLFILENAMES where WHICHFILE is null
--SELECT replace(whichfile,'.csv',''),* FROM dbo.ALLFILENAMES
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%' order by WHICHFILE desc
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql =
'select * into '+ Replace(#filename, '.csv','')+'
from openrowset(''MSDASQL''
,''Driver={Microsoft Access Text Driver (*.txt, *.csv)}''
,''select * from '+#Path+#filename+''')'
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
For Step 1 Maybe you can look at:
http://www.sql-server-performance.com/forum/threads/copying-filenames-to-sql-table.11546/
or
How to list files inside a folder with SQL Server
and then Step 2
How to cast variables in T-SQL for bulk insert?
HTH
You might need to enable the xp_cmdshell first:
sp_configure 'show advanced options', '1'
RECONFIGURE
go
sp_configure 'xp_cmdshell', '1'
RECONFIGURE
go
And, to enable ad_hoc,
sp_configure 'show advanced options', 1;
RECONFIGURE;
GO
sp_configure 'Ad Hoc Distributed Queries', 1;
RECONFIGURE;
GO
To solve step 1, xp_dirtree can also be used to list all files and folders.
Keep in mind that it is an undocumented function. Security precautions must be considered. Intentionally crafted filenames could be an intrusion vector.
In python you can use d6tstack which makes this simple
import d6tstack
import glob
c = d6tstack.combine_csv.CombinerCSV(glob.glob('*.csv'))
c.to_mssql_combine('mssql+pymssql://usr:pwd#localhost/db', 'tablename')
See SQL examples. It also deals with data schema changes, creates table and allows you to preprocess data. It leverages BULK INSERT so should be just as fast.
to expand upon the answer by SarangArd you can replace temp with the following if your file name matches your table name.
' + Left(#filename, Len(#filename)-4) + '
This code will create a new table per CSV file that is imported.
Best to populate empty database from CSV files.
CREATE TABLE ALLFILENAMES
(
WHICHPATH VARCHAR(255)
,WHICHFILE VARCHAR(255)
)
DECLARE #filename VARCHAR(255),
#path VARCHAR(255),
#sql VARCHAR(8000),
#cmd VARCHAR(1000)
SET #path = 'L:\DATA\SOURCE\CSV\' --PATH TO YOUR CSV FILES (CHANGE TO YOUR PATH)
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES
SET WHICHPATH = #path
WHERE WHICHPATH IS NULL
DECLARE c1 CURSOR
FOR SELECT WHICHPATH
,WHICHFILE
FROM ALLFILENAMES
WHERE WHICHFILE LIKE '%.csv%'
OPEN c1
FETCH NEXT FROM c1 INTO #path,
#filename
WHILE ##fetch_status <> -1
BEGIN
CREATE TABLE #Header
(
HeadString NVARCHAR(MAX)
)
DECLARE #Columns NVARCHAR(MAX) = ''
DECLARE #Query NVARCHAR(MAX) = ''
DECLARE #QUERY2 NVARCHAR(MAX) = ''
DECLARE #HeaderQuery NVARCHAR(MAX) = ''
SELECT #HeaderQuery = #HeaderQuery + 'bulk insert #Header from ''' + #path + #filename + '''
with(firstrow=1,lastrow=1)'
EXEC (#HeaderQuery)
SELECT #Columns = (SELECT QUOTENAME(value) + ' nvarchar(max)' + ','
FROM #Header
CROSS APPLY STRING_SPLIT(HeadString,',') FOR xml PATH(''))
IF ISNULL(#Columns,'') <> ''
BEGIN
SET #Columns = LEFT(#Columns,LEN(#Columns) - 1)
SELECT #Query = #Query + 'CREATE TABLE ' + Replace(#filename,'.csv','') + ' (' + replace(#Columns,'"','') + ')'
PRINT #Query
EXEC (#QUERY)
END
SELECT #QUERY2 = #QUERY2 + 'bulk insert ' + replace(Replace(#filename,'.csv',''),'.TPS','') + ' from ''' + #path + #filename + '''
with(firstrow=2,FORMAT=''csv'',FIELDTERMINATOR='','',ROWTERMINATOR=''\n'')'
EXEC (#QUERY2)
DROP TABLE #Header
FETCH NEXT FROM c1 INTO #path,
#filename
END
CLOSE c1
DEALLOCATE c1