So I'm trying to read a bunch of PDFs from a folder into an SQL table, saving them in a varbinary(max) field. This is what I thought would work at first:
CREATE TABLE tempFileName(filnavn VARCHAR(100));
INSERT INTO tempFileName
EXEC xp_cmdshell 'dir /B "C:\temp\Test Folder\"';
--------
DECLARE #path VARCHAR(100) SET #path = 'C:\temp\Test Folder\'
DECLARE #pdf VARBINARY(MAX)
DECLARE #navn varchar(50)
DECLARE #fullpath nvarchar(max)
DECLARE #sql nvarchar(max)
DECLARE c CURSOR FOR
SELECT filnavn
FROM tempFileName
OPEN c
FETCH NEXT FROM c INTO #navn
WHILE(##FETCH_STATUS = 0)
BEGIN
SET #fullpath = #path + #navn
SELECT #pdf = BulkColumn
FROM OPENROWSET(BULK #fullpath, SINGLE_BLOB) AS Document;
--print #sql
INSERT INTO pdftest VALUES(#navn, #pdf)
FETCH NEXT FROM c INTO #navn
END
CLOSE c
DEALLOCATE c
But this doesn't work as it won't allow me to use a variable in this line:
FROM OPENROWSET(BULK #fullpath, SINGLE_BLOB) AS Document;
So I'm pretty sure the trick is to make the whole "select #pdf.." line into a string and then execute it, but I'm not sure how to get the output into a table. I've tried something like this:
SET #fullpath = #path + #navn
SET #sql = 'DECLARE #pdf VARBINARY(MAX) SELECT #pdf = BulkColumn
FROM OPENROWSET(BULK ''' + #fullpath + ''' , SINGLE_BLOB) AS Document;'
--print #sql
--SELECT #pdf, DATALENGTH(#pdf)
--INSERT INTO pdftest VALUES(#navn, #pdf)
EXEC sp_executesql #sql, N'#fil varbinary(max) out', #fil out
But the #fil variable is just empty after this. How do I best go about getting these files into a table?
Why not just skip the variable assignment of the SELECT in your loop and use the OPENROWSET function inside your INSERT? The general idea:
INSERT INTO pdftest SELECT #navn, * FROM OPENROWSET(BULK, 'C:\thefile.txt', SINGLE_BLOB) AS document
And of course turn above into dynamic SQL. I'll probably get a few single-quotes wrong here, but again the general idea:
SET #sql =
'INSERT INTO pdftest
SELECT '' + #navn + '', *
FROM OPENROWSET(BULK, ''' + #fullpath + ''', SINGLE_BLOB) AS document
'
Related
I'm using T-SQL. The goal is to insert multiples files into a database.
If I'm using without a loop, it's working fine.
In the loop, I always get this error:
#InputXML should be declared
My code:
IF OBJECT_ID('TEMPDB..#TEMP_FILES') IS NOT NULL
DROP TABLE #TEMP_FILES
CREATE TABLE #TEMP_FILES
(
FileName VARCHAR(MAX),
DEPTH VARCHAR(MAX),
[FILE] VARCHAR(MAX)
)
INSERT INTO #TEMP_FILES
EXEC master.dbo.xp_DirTree '\\MyServer\MyFolder\',1,1
DELETE FROM #TEMP_FILES WHERE RIGHT(FileName,4) != '.XML'
--
SET QUOTED_IDENTIFIER ON
GO
TRUNCATE Table MyTable2
DECLARE #InputXML XML
DECLARE #FILENAME VARCHAR(MAX),#SQL VARCHAR(MAX)
WHILE EXISTS(SELECT * FROM #TEMP_FILES)
BEGIN
SET #FILENAME = (SELECT TOP 1 FileName FROM #TEMP_FILES)
SET #sql = 'SELECT #InputXML = CAST(x AS XML) FROM OPENROWSET(BULK \\MyServer\MyFolder\'''+ #FILENAME +''', SINGLE_BLOB) AS T(x)
INSERT INTO MyTable2 ([id],[version], [name], [listId], [listCode])
SELECT
product.value(''(#id)[1]'', ''NVARCHAR(10)''),
product.value(''(#version)[1]'', ''NVARCHAR(14)''),
product.value(''(name[1])'', ''NVARCHAR(255)''),
product.value(''(listId[1])'', ''NVARCHAR(9)''),
product.value(''(listCode[1])'', ''NVARCHAR(10)'')
FROM #InputXML.nodes(''xxx/values/value'') AS X(product)'
EXEC(#SQL)
DELETE FROM #TEMP_FILES
WHERE FileName = #FILENAME
END
You need to declare the variable inside the dynamic SQL (which should be nvarchar not varchar). You should also use QUOTENAME to ensure no issues with the filename:
DECLARE #sql nvarchar(max) = N'
DECLARE #InputXML XML;
SELECT #InputXML = CAST(x AS XML) FROM OPENROWSET(BULK ' + QUOTENAME(N'\\MyServer\MyFolder\' + #FILENAME, '''') + N', SINGLE_BLOB) AS T(x)
INSERT INTO MyTable2 ([id],[version], [name], [listId], [listCode])
SELECT
product.value(''(#id)[1]'', ''NVARCHAR(10)''),
product.value(''(#version)[1]'', ''NVARCHAR(14)''),
product.value(''(name[1])'', ''NVARCHAR(255)''),
product.value(''(listId[1])'', ''NVARCHAR(9)''),
product.value(''(listCode[1])'', ''NVARCHAR(10)'')
FROM #InputXML.nodes(''xxx/values/value'') AS X(product)'
I will say though, that I urge you to find another method to load files into SQL Server. Dynamic OPENROWSET, especially from user input, is not advisable. Bulk Insert or BCP may be an option.
I am trying to import 24 .txt files to the server but I have limited privileges. All files start with the same characters, but '*' this masking cannot be used for this query. I use below script.
SELECT * into TABLE
FROM OPENROWSET( BULK 'c:\path\files\good*.txt', FORMATFILE = 'c:\path\files\import.xml',FIRSTROW = 2) AS DATA;
You would need to create the table first and then insert each file using a loop.
try the following script after creating the destination table:
declare #filepath varchar(100)= 'c:\path\files\'
,#pattern varchar(100)= 'good*.txt'
,#TableName varchar(100)= 'TestTable'
DECLARE #query varchar(1000)
DECLARE #numfiles int
DECLARE #filename varchar(100)
DECLARE #files TABLE (SourceFileName varchar(200) NULL)
SET #query = 'master.dbo.xp_cmdshell "dir ' + #filepath+#pattern + ' /b"'
INSERT #files(SourceFileName)
EXEC (#query)
DECLARE CUR CURSOR FAST_FORWARD FOR
SELECT SourceFileName FROM #files WHERE SourceFileName IS NOT NULL
SET #numfiles =0
OPEN CUR
FETCH NEXT FROM CUR INTO #filename
WHILE (##FETCH_STATUS = 0)
BEGIN
print #filename
SET #numfiles+=1
SET #query = ('BULK INSERT ' + #TableName
+ ' FROM ''' + #Filepath+#filename + ''' WITH(
FORMATFILE = ''c:\path\files\import.xml'',
FIRSTROW = 2
);'
)
PRINT #query
EXEC (#query)
FETCH NEXT FROM CUR INTO #filename
END
CLOSE CUR
DEALLOCATE CUR
I am trying to execute this query:
declare #tablename varchar(50)
set #tablename = 'test'
select * from #tablename
This produces the following error:
Msg 1087, Level 16, State 1, Line 5
Must declare the table variable "#tablename".
What's the right way to have the table name populated dynamically?
For static queries, like the one in your question, table names and column names need to be static.
For dynamic queries, you should generate the full SQL dynamically, and use sp_executesql to execute it.
Here is an example of a script used to compare data between the same tables of different databases:
Static query:
SELECT * FROM [DB_ONE].[dbo].[ACTY]
EXCEPT
SELECT * FROM [DB_TWO].[dbo].[ACTY]
Since I want to easily change the name of table and schema, I have created this dynamic query:
declare #schema sysname;
declare #table sysname;
declare #query nvarchar(max);
set #schema = 'dbo'
set #table = 'ACTY'
set #query = '
SELECT * FROM [DB_ONE].' + QUOTENAME(#schema) + '.' + QUOTENAME(#table) + '
EXCEPT
SELECT * FROM [DB_TWO].' + QUOTENAME(#schema) + '.' + QUOTENAME(#table);
EXEC sp_executesql #query
Since dynamic queries have many details that need to be considered and they are hard to maintain, I recommend that you read: The curse and blessings of dynamic SQL
Change your last statement to this:
EXEC('SELECT * FROM ' + #tablename)
This is how I do mine in a stored procedure. The first block will declare the variable, and set the table name based on the current year and month name, in this case TEST_2012OCTOBER. I then check if it exists in the database already, and remove if it does. Then the next block will use a SELECT INTO statement to create the table and populate it with records from another table with parameters.
--DECLARE TABLE NAME VARIABLE DYNAMICALLY
DECLARE #table_name varchar(max)
SET #table_name =
(SELECT 'TEST_'
+ DATENAME(YEAR,GETDATE())
+ UPPER(DATENAME(MONTH,GETDATE())) )
--DROP THE TABLE IF IT ALREADY EXISTS
IF EXISTS(SELECT name
FROM sysobjects
WHERE name = #table_name AND xtype = 'U')
BEGIN
EXEC('drop table ' + #table_name)
END
--CREATES TABLE FROM DYNAMIC VARIABLE AND INSERTS ROWS FROM ANOTHER TABLE
EXEC('SELECT * INTO ' + #table_name + ' FROM dbo.MASTER WHERE STATUS_CD = ''A''')
Use:
CREATE PROCEDURE [dbo].[GetByName]
#TableName NVARCHAR(100)
AS
BEGIN
-- SET NOCOUNT ON added to prevent extra result sets from
-- interfering with SELECT statements.
SET NOCOUNT ON;
DECLARE #sSQL nvarchar(500);
SELECT #sSQL = N'SELECT * FROM' + QUOTENAME(#TableName);
EXEC sp_executesql #sSQL
END
You can't use a table name for a variable. You'd have to do this instead:
DECLARE #sqlCommand varchar(1000)
SET #sqlCommand = 'SELECT * from yourtable'
EXEC (#sqlCommand)
You'll need to generate the SQL content dynamically:
declare #tablename varchar(50)
set #tablename = 'test'
declare #sql varchar(500)
set #sql = 'select * from ' + #tablename
exec (#sql)
Use sp_executesql to execute any SQL, e.g.
DECLARE #tbl sysname,
#sql nvarchar(4000),
#params nvarchar(4000),
#count int
DECLARE tblcur CURSOR STATIC LOCAL FOR
SELECT object_name(id) FROM syscolumns WHERE name = 'LastUpdated'
ORDER BY 1
OPEN tblcur
WHILE 1 = 1
BEGIN
FETCH tblcur INTO #tbl
IF ##fetch_status <> 0
BREAK
SELECT #sql =
N' SELECT #cnt = COUNT(*) FROM dbo.' + quotename(#tbl) +
N' WHERE LastUpdated BETWEEN #fromdate AND ' +
N' coalesce(#todate, ''99991231'')'
SELECT #params = N'#fromdate datetime, ' +
N'#todate datetime = NULL, ' +
N'#cnt int OUTPUT'
EXEC sp_executesql #sql, #params, '20060101', #cnt = #count OUTPUT
PRINT #tbl + ': ' + convert(varchar(10), #count) + ' modified rows.'
END
DEALLOCATE tblcur
You need to use the SQL Server dynamic SQL:
DECLARE #table NVARCHAR(128),
#sql NVARCHAR(MAX);
SET #table = N'tableName';
SET #sql = N'SELECT * FROM ' + #table;
Use EXEC to execute any SQL:
EXEC (#sql)
Use EXEC sp_executesql to execute any SQL:
EXEC sp_executesql #sql;
Use EXECUTE sp_executesql to execute any SQL:
EXECUTE sp_executesql #sql
Declare #tablename varchar(50)
set #tablename = 'Your table Name'
EXEC('select * from ' + #tablename)
Also, you can use this...
DECLARE #SeqID varchar(150);
DECLARE #TableName varchar(150);
SET #TableName = (Select TableName from Table);
SET #SeqID = 'SELECT NEXT VALUE FOR ' + #TableName + '_Data'
exec (#SeqID)
Declare #fs_e int, #C_Tables CURSOR, #Table varchar(50)
SET #C_Tables = CURSOR FOR
select name from sysobjects where OBJECTPROPERTY(id, N'IsUserTable') = 1 AND name like 'TR_%'
OPEN #C_Tables
FETCH #C_Tables INTO #Table
SELECT #fs_e = sdec.fetch_Status FROM sys.dm_exec_cursors(0) as sdec where sdec.name = '#C_Tables'
WHILE ( #fs_e <> -1)
BEGIN
exec('Select * from ' + #Table)
FETCH #C_Tables INTO #Table
SELECT #fs_e = sdec.fetch_Status FROM sys.dm_exec_cursors(0) as sdec where sdec.name = '#C_Tables'
END
I am trying to insert file through SQL. I use following query.
INSERT INTO [dbo].[Attachments] (FileName, FileBinary)
SELECT
'non-date-in-sql-server-column',
BulkColumn
FROM
OPENROWSET(Bulk 'C:\Users\Pictures\Picture.JPG', SINGLE_BLOB) AS BLOB
It's working fine.
I want to write the procedure that take dynamic path. Its giving me error that I cannot take Filebinary in addin. Which is datatype varbinary. What is the best way to do ?
I have done following but its not taking properly binary value.
DECLARE #SQLString NVARCHAR(MAX)
SET #SQLString = 'SELECT ' + '''' +#Filename +'''' + ' AS Name,' + 'FileBinary
FROM OPENROWSET(BULK N''' + #ImagePath + ''',SINGLE_BLOB) AS FileBinary(FileBinary);'
Insert Into Attachments (ApplicantID, FileName, FileBinary)
Values (#ApplicantID, #FileName, Convert(varbinary(max), #SQLString))
Put the Insert statement inside a dynamic query and execute it.
Now your #SQLString will not have the FileBinary value it will have the dynamically framed string . You need to execute it to get the values
DECLARE #SQLString NVARCHAR(MAX),
#Filename VARCHAR(500), -- Pass file name here
#ApplicantID VARCHAR(500) --Pass Application ID here
SET #SQLString = '
Insert Into Attachments
(
ApplicantID,
FileName,
FileBinary
)
SELECT #ApplicantID,#Filename,FileBinary
FROM OPENROWSET(BULK N''' + #ImagePath
+ ''',SINGLE_BLOB) AS FileBinary(FileBinary);'
EXEC Sp_executesql
#SQLString,
N'#Filename varchar(500),#ApplicantID varchar(500)',
#Filename =#Filename,
#ApplicantID=#ApplicantID
I have a folder called "Dump." This folder consists of various .CSV Files.
The folder Location is 'C:\Dump'
I want to Import the contents of these files into SQL Server.
I want the rough code along with proper comments so that I understand it.
I have tried a few codes that I found on the Net. But they haven't quite worked out for me for some strange reason.
The steps I would like to have are
Step 1: Copy all the File Names in the folder to a Table
Step 2: Iterate through the table and copy the data from the files using Bulk Insert.
Someone do please help me out on this one. Thanks a lot in advance :)
--BULK INSERT MULTIPLE FILES From a Folder
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'C:\Dump\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%'
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql = 'BULK INSERT Temp FROM ''' + #path + #filename + ''' '
+ ' WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
) '
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
--Extras
--delete from ALLFILENAMES where WHICHFILE is NULL
--select * from ALLFILENAMES
--drop table ALLFILENAMES
This will give you separate tables for each file.
--BULK INSERT MULTIPLE FILES From a Folder
drop table allfilenames
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'D:\Benihana\backup_csv_benihana_20191128032207_part_1\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
delete from ALLFILENAMES where WHICHFILE is null
--SELECT replace(whichfile,'.csv',''),* FROM dbo.ALLFILENAMES
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%' order by WHICHFILE desc
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql =
'select * into '+ Replace(#filename, '.csv','')+'
from openrowset(''MSDASQL''
,''Driver={Microsoft Access Text Driver (*.txt, *.csv)}''
,''select * from '+#Path+#filename+''')'
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
For Step 1 Maybe you can look at:
http://www.sql-server-performance.com/forum/threads/copying-filenames-to-sql-table.11546/
or
How to list files inside a folder with SQL Server
and then Step 2
How to cast variables in T-SQL for bulk insert?
HTH
You might need to enable the xp_cmdshell first:
sp_configure 'show advanced options', '1'
RECONFIGURE
go
sp_configure 'xp_cmdshell', '1'
RECONFIGURE
go
And, to enable ad_hoc,
sp_configure 'show advanced options', 1;
RECONFIGURE;
GO
sp_configure 'Ad Hoc Distributed Queries', 1;
RECONFIGURE;
GO
To solve step 1, xp_dirtree can also be used to list all files and folders.
Keep in mind that it is an undocumented function. Security precautions must be considered. Intentionally crafted filenames could be an intrusion vector.
In python you can use d6tstack which makes this simple
import d6tstack
import glob
c = d6tstack.combine_csv.CombinerCSV(glob.glob('*.csv'))
c.to_mssql_combine('mssql+pymssql://usr:pwd#localhost/db', 'tablename')
See SQL examples. It also deals with data schema changes, creates table and allows you to preprocess data. It leverages BULK INSERT so should be just as fast.
to expand upon the answer by SarangArd you can replace temp with the following if your file name matches your table name.
' + Left(#filename, Len(#filename)-4) + '
This code will create a new table per CSV file that is imported.
Best to populate empty database from CSV files.
CREATE TABLE ALLFILENAMES
(
WHICHPATH VARCHAR(255)
,WHICHFILE VARCHAR(255)
)
DECLARE #filename VARCHAR(255),
#path VARCHAR(255),
#sql VARCHAR(8000),
#cmd VARCHAR(1000)
SET #path = 'L:\DATA\SOURCE\CSV\' --PATH TO YOUR CSV FILES (CHANGE TO YOUR PATH)
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES
SET WHICHPATH = #path
WHERE WHICHPATH IS NULL
DECLARE c1 CURSOR
FOR SELECT WHICHPATH
,WHICHFILE
FROM ALLFILENAMES
WHERE WHICHFILE LIKE '%.csv%'
OPEN c1
FETCH NEXT FROM c1 INTO #path,
#filename
WHILE ##fetch_status <> -1
BEGIN
CREATE TABLE #Header
(
HeadString NVARCHAR(MAX)
)
DECLARE #Columns NVARCHAR(MAX) = ''
DECLARE #Query NVARCHAR(MAX) = ''
DECLARE #QUERY2 NVARCHAR(MAX) = ''
DECLARE #HeaderQuery NVARCHAR(MAX) = ''
SELECT #HeaderQuery = #HeaderQuery + 'bulk insert #Header from ''' + #path + #filename + '''
with(firstrow=1,lastrow=1)'
EXEC (#HeaderQuery)
SELECT #Columns = (SELECT QUOTENAME(value) + ' nvarchar(max)' + ','
FROM #Header
CROSS APPLY STRING_SPLIT(HeadString,',') FOR xml PATH(''))
IF ISNULL(#Columns,'') <> ''
BEGIN
SET #Columns = LEFT(#Columns,LEN(#Columns) - 1)
SELECT #Query = #Query + 'CREATE TABLE ' + Replace(#filename,'.csv','') + ' (' + replace(#Columns,'"','') + ')'
PRINT #Query
EXEC (#QUERY)
END
SELECT #QUERY2 = #QUERY2 + 'bulk insert ' + replace(Replace(#filename,'.csv',''),'.TPS','') + ' from ''' + #path + #filename + '''
with(firstrow=2,FORMAT=''csv'',FIELDTERMINATOR='','',ROWTERMINATOR=''\n'')'
EXEC (#QUERY2)
DROP TABLE #Header
FETCH NEXT FROM c1 INTO #path,
#filename
END
CLOSE c1
DEALLOCATE c1