I am using T-SQL
I have a few excel files located here: C:\MyFiles\
I want to remove all the apostrophes in the file names in that directory.
Now to remove apostrophes one would use code like this.
update MyTable
set FileName= replace(FileName, '''', '')
If I had all the file name in the DB it would be easy to do with the code above. But I need to update the file names that are located on disk.
How would I go about doing this with T-SQL?
It must be T-SQL because I need to add it to my existing code in my Stored Procedure.
SET NOCOUNT ON;
CREATE TABLE #FileList
(
FileID INT IDENTITY(1, 1)
,Line VARCHAR(512)
)
CREATE TABLE #temp
(
isFileThere BIT
,isDirectory BIT
,parentDirExists BIT
)
DECLARE #Command VARCHAR(1024)
, #RowCount INT
, #counter INT
, #FileName VARCHAR(1024)
, #FileExists BIT
SET #Command = 'dir C:\MyFiles\ /A-D /B'
PRINT #Command
INSERT #FileList
EXEC master.dbo.xp_cmdshell #Command
DELETE FROM #FileList
WHERE Line IS NULL
SELECT #RowCount = COUNT(*)
FROM [#FileList]
SET #counter = 1
WHILE ( #counter <= #RowCount )
BEGIN
SELECT #FileName = [Line]
FROM [#FileList]
WHERE [FileID] = #counter
SET #Command = 'C:\MyFiles\' + #FileName + ''
PRINT #Command
INSERT [#temp]
EXEC master.dbo.xp_fileExist #Command
SELECT #FileExists = [isFileThere]
FROM [#temp]
IF #FileExists = 1
AND CHARINDEX('''', #FileName) > 0
SET #Command = 'REN "C:\MyFiles\' + #FileName + '" "'
+ REPLACE(#FileName, '''', '') + '"'
ELSE
SET #Command = ''
SET #counter = #counter + 1
PRINT #Command
IF LEN(#Command) > 0
EXEC master.dbo.xp_cmdshell #Command
END
DROP TABLE #FileList
DROP TABLE [#temp]
First you need to enable xp_cmdshell
EXEC sp_configure 'show advanced options', 1
GO
RECONFIGURE
GO
EXEC sp_configure 'xp_cmdshell', 1
GO
RECONFIGURE
GO
Then you can use sp_cmdshell to retrieve the files names from the directory
Declare #Directory TABLE (Files Varchar(MAX))
Declare #File TABLE (Name varchar(MAX))
INSERT INTO #Directory
EXEC XP_CMDSHELL 'DIR "D:"'
Insert into #File
Select reverse(LEFT(reverse(Files),charindex(' ' ,reverse(Files)))) from #Directory
Select * from #FILE
Now you get the file names in the table variable #FILE and use function like replace or your own custom function to replace apostrophes with the exact filename
Try this it will work for you
1) Create a sp as mentioned below
CREATE PROCEDURE dbo.ListPathsXML
#FileSpec VARCHAR(2000),
#order VARCHAR (80) = '/O-D',--sort by date time oldest first
#xmlFileList XML OUTPUT
AS
DECLARE #myfiles TABLE (MyID INT IDENTITY(1,1) PRIMARY KEY, FullPath VARCHAR(2000))
DECLARE #CommandLine VARCHAR(4000)
IF #order IS NOT NULL -- abort if the order is silly
BEGIN
SELECT #CommandLine =LEFT('dir "' + #FileSpec + '" /A-D /B /S '+#order,4000)
INSERT INTO #MyFiles (FullPath)
EXECUTE xp_cmdshell #CommandLine
DELETE FROM #MyFiles WHERE fullpath IS NULL
OR fullpath = 'File Not Found'
END
SET #xmlFileList = (SELECT fullpath FROM #MyFiles
FOR
XML PATH('thefile'),
ROOT('thefiles'),
TYPE)
2) and then give a name of directory where you want to replace the name of files
DECLARE #LotsOfText NVARCHAR(MAX),
#ii INT,
#iiMax INT,
#File VARCHAR(2000),
#Command NVARCHAR(4000)
DECLARE #files TABLE (MyID INT IDENTITY(1,1) PRIMARY KEY, [Path] VARCHAR(2000))
DECLARE #FileList XML
EXECUTE ListPathsXML 'D:\QAconfig\',
DEFAULT , #XMLFileList = #FileList OUTPUT
INSERT INTO #files(path)
SELECT x.thefile.value('fullpath[1]', 'varchar(2000)') AS [path]
FROM #FileList.nodes('//thefiles/thefile') AS x ( thefile )
--don't look at the current errorlog!
SELECT #ii=1, #iiMax=MAX(MyID) FROM #Files
WHILE #ii<=#iiMax
BEGIN
SELECT #File= [path] FROM #files WHERE MyID=#ii
print #File
SELECT #command='EXEC master..xp_cmdshell' + '''MOVE '+ Replace(#FILE,'''','''''') + ' ' +REPLACE(#FILE,'''','') +''''
print #command
EXECUTE sp_ExecuteSQL #command--, N'#lotsOfText nvarchar(max) output ',#lotsoftext output
SELECT #ii=#ii+1
END
Related
I am trying to import .txt files daily into sql server 2008 table and I want to automate it
so in steps:
1- I receive 2 files daily with name hazem.log.date and hazem.log.date2
2- I need to have a way to import them daily and automatically
3- I will use the job, but which command or query should be used in this case?
Try running below through SQLServer jobs..
BULK INSERT dbo.ImportTest
FROM 'C:\ImportData.txt' --replace name of your files
WITH ( FIELDTERMINATOR =',', FIRSTROW = 2 )
you also can use
bcp dbo.ImportTest in 'C:\ImportData.txt' -T -SserverName\instanceName
For Mutiple files..you can do like this..
1.Create a stored proc first..
Create procedure usp_ImportMultipleFilesBCP #servername varchar(128),
#DatabaseName varchar(128), #filepath varchar(500), #pattern varchar(100),
#TableName varchar(128)
as
declare #query varchar(1000)
declare #max1 int
declare #count1 int
Declare #filename varchar(100)
set #count1 =0
create table #x (name varchar(200))
set #query ='master.dbo.xp_cmdshell "dir '+#filepath+#pattern +' /b"'
insert #x exec (#query)
delete from #x where name is NULL
select identity(int,1,1) as ID, name into #y from #x
drop table #x
set #max1 = (select max(ID) from #y)
--print #max1
--print #count1
--select * from #y
While #count1 <= #max1
begin
set #count1=#count1+1
set #filename = (select name from #y where [id] = #count1)
set #Query ='bcp "'+ #databasename+'.dbo.'+#Tablename + '"
in "'+ #Filepath+#Filename+'" -S' + #servername + ' -T -c -r\n -t,'
set #Query = 'MASTER.DBO.xp_cmdshell '+ "'"+ #query +"'"
--print #query
EXEC ( #query)
insert into logtable (query) select #query
end
2.Now run above sp to import all files of desired extension
Exec usp_ImportMultipleFilesBCP 'SQL','Bank','c:\Myimport\','*.csv','Account'--table account
Note:
You will need to enable Xp_cmdshell
References:
https://www.mssqltips.com/sqlservertip/1207/different-options-for-importing-data-into-sql-server/
http://www.databasejournal.com/features/mssql/article.php/3325701/Import-multiple-Files-to-SQL-Server-using-T-SQL.htm
DECLARE #SQL_BULK VARCHAR(MAX)
SET #SQL_BULK = 'BULK INSERT [dbo].[Table]
FROM ''' + #BatchFileName + '''
WITH
(
FIRSTROW = 2,
FIELDTERMINATOR = ''\t'',
ROWTERMINATOR = ''0x0a''
)'
EXEC (#SQL_BULK)
I have this code to do a bulk load. Works fine, but I would like to have the #BatchFileName also in there as a column (each row contains the same value).
Is this possible during the bulk load? Or how can I add it later on in a separate function?
Thanks in advance
i use this script to iterate and bulkinsert files in directory:
create table #x (name varchar(200))
DECLARE #query varchar (1000),#conta int ,#query2 varchar (1000),#NOME varchar(50)
set #conta=1
set #query ='master.dbo.xp_cmdshell "dir '+'C:\directoryname'+'*.csv' +' /b"'
insert #x exec (#query)
delete from #x where name is NULL
select identity(int,1,1) as ID, name into #y from #x
drop table #x
WHILE #conta<221 --number of files
BEGIN
SELECT #NOME=name FROM #y WHERE ID=#conta
set #Query2 ='BULK INSERT [dbo].[tablename] FROM '+ '''C:\directoryname'+#NOME+'''
WITH (
FIELDTERMINATOR = '','',ROWTERMINATOR = ''0x0a'')'
SELECT #query2
exec (#Query2)
set #conta=#conta+1
END
drop table #y
I have sp in MSSQL server - code below. When I run it from job, or SSMS it runs OK. But I need to run it from VB6 app with ADODB.
My VB6 code:
Dim cmd As New ADODB.Command
cmd.ActiveConnection = CNN
cmd.CommandTimeout = 180
cmd.CommandText = "dbbackup"
cmd.CommandType = ADODB.CommandTypeEnum.adCmdStoredProc
cmd.Execute(, , ADODB.ConnectOptionEnum.adAsyncConnect)
Problem is: When database backup is almost done - about 90+%, cmd.State changes from Executing to Closed and VB6 code continue in executing (to this moment it waits for sp to complete). But there is a lot of code after backup which never run this way(old backup delete,...). I realized that “Last database backup” property on MSSQL database was not set and in table msdb.dbo.backupset there are no rows for my backup. But there si good restorable backup on HDD.
When i stops program for 5 minutes in debug, sp runs properly to end and everything is OK. This backup code is last code in app run and after it ends program closes all connections and exits. I added wait to VB6 code and it helps on some servers, but many other servers still has same problem.
I think main question is why MSSQL server returns control flow to VB6 code and sp is not completed yet.
Thanks
sp code:
PROCEDURE [dbo].[dbBackup]
AS
BEGIN
SET NOCOUNT ON;
EXEC sp_configure 'show advanced options', 1
RECONFIGURE
EXEC sp_configure 'xp_cmdshell', 1
RECONFIGURE
If OBJECT_ID('tempdb..#DBName','u') IS NULL
Create Table #DBName
(
ID int identity (1,1) ,
Name varchar(128) not null ,
RetentionPeriod int null,
BackupPath varchar(255) default(''),
DBSize float default(0)
)
If OBJECT_ID('tempdb..#ExistingBackups', 'u') IS NULL
Create Table #ExistingBackups
(
Name varchar(128) ,
ID int identity (1,1)
)
Declare #Path varchar(255)
Declare #sql varchar(1000)
Declare #Name varchar(128)
Declare #RetentionPeriod int
Declare #LastBackupToKeep varchar(8)
Declare #ID int
Declare #MaxID int
Declare #eName varchar(255)
Declare #eMaxID int
Declare #eID int
Declare #eTimeStamp varchar(20)
Declare #errMsg nvarchar(2048)
Declare #errCount int; set #errCount = 0;
Declare #freeSpace bigint
Declare #pageSize float
Declare #dbSize bigint
Declare #procDate datetime
Declare #Sklad char(3)
Declare #backupName as varchar(255)
Select #pageSize = v.low / 1024 From master..spt_values v (noLock) Where v.number = 1 And v.[type] = 'E'
Select Top 1 #sklad = sklad_id From dbo.pohyb (noLock) Where Convert(int, sklad_id) > 500
Set #procDate = GETDATE()
Truncate Table #DBName
Insert Into #DBName (Name, RetentionPeriod, BackupPath)
Select DBName, BackupsToStore, BackupPath
From dbo.databaseBackup (noLock)
Where runBackup = 1
Select #MaxID = max(ID), #ID = 0 From #DBName
While #ID < #MaxID
Begin
Select #ID = min(ID) From #DBName Where ID > #ID
Select #Name = Name, #RetentionPeriod = RetentionPeriod, #Path = BackupPath
From #DBName
Where ID = #ID
If SUBSTRING(#Path, Len(#Path), 1) <> '\' Set #Path = #Path + '\'
Set #sql = 'Update #DBName Set DBSize= (Select Round(Sum(size) *' + CONVERT(varchar, #pageSize) + '/1024, 0) From ' + #Name + '.dbo.sysfiles (noLock)) Where Name = ''' + #Name + ''''
Exec (#sql)
Select #dbSize = DBSize From #DBName
--Exec #freeSpace = dbo.getDiskFreeSpace #drive = #Path
--If #freeSpace > #dbSize
--Begin
Set #eTimeStamp = REPLACE(REPLACE(CONVERT(varchar, #procDate, 113), ' ', '_'), ':', '-')
Set #sql = #Path + #Name + '_' + #eTimeStamp + '.bak'
Set #errMsg = 'OK'
Begin Try
SET #backupName = 'Objednavky backup by job ' + CONVERT(varchar, GETDATE(), 104) + ' ' + CONVERT(varchar, GETDATE(), 108);
Backup Database #Name To Disk = #sql
WITH NAME = #backupName;
-------mazanie backupu begin
Truncate Table #ExistingBackups
Set #sql = 'dir /B /OD ' + #Path + #Name + '_*.bak'
Insert #ExistingBackups Exec master..xp_cmdshell #sql
If Exists (Select 1 From #ExistingBackups Where PATINDEX('%File Not Found%', Name) > 0)
Truncate Table #ExistingBackups
Delete From #ExistingBackups Where Name IS NULL
Select #eID = 0
Select #eMaxID = Max(ID) - #RetentionPeriod From #ExistingBackups
While #eID < #eMaxID
Begin
Select #eID = Min(ID) From #ExistingBackups Where ID > #eID
Select #eName = Name From #ExistingBackups Where ID = #eID
Set #sql = 'del ' + #Path + #eName
Exec master..xp_cmdshell #sql
End
Truncate Table #ExistingBackups
-------mazanie backupu end
End Try
Begin Catch
Set #errMsg = #errMsg + '||' + CONVERT(varchar,ERROR_MESSAGE())
Set #errCount = #errCount + 1;
End Catch
--End
--Else
--Set #errMsg = 'Pln? disk (Vo?n? miesto: ' + CONVERT(varchar, #freeSpace) + ' MB, potrebn? aspo?: ' + CONVERT(varchar, #dbSize) + ' MB)'
Insert Into [dbo].[databaseBackup_log] ([Sklad_id], [DBName], [BackupDate], [Status]) Values (#Sklad, #Name, #procDate, Ltrim(Rtrim(CONVERT(varchar,#errMsg))))
End
Drop Table #DBName
Drop Table #ExistingBackups
IF #errCount > 0 BEGIN
RAISERROR (#errMsg, 16, 2) WITH SETERROR
END
RETURN 0;
END
I have a folder called "Dump." This folder consists of various .CSV Files.
The folder Location is 'C:\Dump'
I want to Import the contents of these files into SQL Server.
I want the rough code along with proper comments so that I understand it.
I have tried a few codes that I found on the Net. But they haven't quite worked out for me for some strange reason.
The steps I would like to have are
Step 1: Copy all the File Names in the folder to a Table
Step 2: Iterate through the table and copy the data from the files using Bulk Insert.
Someone do please help me out on this one. Thanks a lot in advance :)
--BULK INSERT MULTIPLE FILES From a Folder
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'C:\Dump\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%'
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql = 'BULK INSERT Temp FROM ''' + #path + #filename + ''' '
+ ' WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
) '
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
--Extras
--delete from ALLFILENAMES where WHICHFILE is NULL
--select * from ALLFILENAMES
--drop table ALLFILENAMES
This will give you separate tables for each file.
--BULK INSERT MULTIPLE FILES From a Folder
drop table allfilenames
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'D:\Benihana\backup_csv_benihana_20191128032207_part_1\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
delete from ALLFILENAMES where WHICHFILE is null
--SELECT replace(whichfile,'.csv',''),* FROM dbo.ALLFILENAMES
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%' order by WHICHFILE desc
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql =
'select * into '+ Replace(#filename, '.csv','')+'
from openrowset(''MSDASQL''
,''Driver={Microsoft Access Text Driver (*.txt, *.csv)}''
,''select * from '+#Path+#filename+''')'
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
For Step 1 Maybe you can look at:
http://www.sql-server-performance.com/forum/threads/copying-filenames-to-sql-table.11546/
or
How to list files inside a folder with SQL Server
and then Step 2
How to cast variables in T-SQL for bulk insert?
HTH
You might need to enable the xp_cmdshell first:
sp_configure 'show advanced options', '1'
RECONFIGURE
go
sp_configure 'xp_cmdshell', '1'
RECONFIGURE
go
And, to enable ad_hoc,
sp_configure 'show advanced options', 1;
RECONFIGURE;
GO
sp_configure 'Ad Hoc Distributed Queries', 1;
RECONFIGURE;
GO
To solve step 1, xp_dirtree can also be used to list all files and folders.
Keep in mind that it is an undocumented function. Security precautions must be considered. Intentionally crafted filenames could be an intrusion vector.
In python you can use d6tstack which makes this simple
import d6tstack
import glob
c = d6tstack.combine_csv.CombinerCSV(glob.glob('*.csv'))
c.to_mssql_combine('mssql+pymssql://usr:pwd#localhost/db', 'tablename')
See SQL examples. It also deals with data schema changes, creates table and allows you to preprocess data. It leverages BULK INSERT so should be just as fast.
to expand upon the answer by SarangArd you can replace temp with the following if your file name matches your table name.
' + Left(#filename, Len(#filename)-4) + '
This code will create a new table per CSV file that is imported.
Best to populate empty database from CSV files.
CREATE TABLE ALLFILENAMES
(
WHICHPATH VARCHAR(255)
,WHICHFILE VARCHAR(255)
)
DECLARE #filename VARCHAR(255),
#path VARCHAR(255),
#sql VARCHAR(8000),
#cmd VARCHAR(1000)
SET #path = 'L:\DATA\SOURCE\CSV\' --PATH TO YOUR CSV FILES (CHANGE TO YOUR PATH)
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES
SET WHICHPATH = #path
WHERE WHICHPATH IS NULL
DECLARE c1 CURSOR
FOR SELECT WHICHPATH
,WHICHFILE
FROM ALLFILENAMES
WHERE WHICHFILE LIKE '%.csv%'
OPEN c1
FETCH NEXT FROM c1 INTO #path,
#filename
WHILE ##fetch_status <> -1
BEGIN
CREATE TABLE #Header
(
HeadString NVARCHAR(MAX)
)
DECLARE #Columns NVARCHAR(MAX) = ''
DECLARE #Query NVARCHAR(MAX) = ''
DECLARE #QUERY2 NVARCHAR(MAX) = ''
DECLARE #HeaderQuery NVARCHAR(MAX) = ''
SELECT #HeaderQuery = #HeaderQuery + 'bulk insert #Header from ''' + #path + #filename + '''
with(firstrow=1,lastrow=1)'
EXEC (#HeaderQuery)
SELECT #Columns = (SELECT QUOTENAME(value) + ' nvarchar(max)' + ','
FROM #Header
CROSS APPLY STRING_SPLIT(HeadString,',') FOR xml PATH(''))
IF ISNULL(#Columns,'') <> ''
BEGIN
SET #Columns = LEFT(#Columns,LEN(#Columns) - 1)
SELECT #Query = #Query + 'CREATE TABLE ' + Replace(#filename,'.csv','') + ' (' + replace(#Columns,'"','') + ')'
PRINT #Query
EXEC (#QUERY)
END
SELECT #QUERY2 = #QUERY2 + 'bulk insert ' + replace(Replace(#filename,'.csv',''),'.TPS','') + ' from ''' + #path + #filename + '''
with(firstrow=2,FORMAT=''csv'',FIELDTERMINATOR='','',ROWTERMINATOR=''\n'')'
EXEC (#QUERY2)
DROP TABLE #Header
FETCH NEXT FROM c1 INTO #path,
#filename
END
CLOSE c1
DEALLOCATE c1
What I have
I am currently inserting all the SQL files from the current directory in a temporary table :
create table #tmpSQLFiles(SQLFileName varchar(max))
set #SqlStatement = 'master.dbo.xp_cmdshell ''dir /b "' + #DirPath + '*.sql"'''
insert into #tmpSQLFiles
execute (#SqlStatement)
This works well.
What I want
Now I want the table to have an extra varchar column in which I can store a certain desired string :
create table #tmpSQLFiles(SQLFileName varchar(max), MyString varchar(max))
set #SqlStatement = 'master.dbo.xp_cmdshell ''dir /b "' + #DirPath+ '*.sql"'''
What I have tried
How can I do the insert? I have tried :
insert into #tmpSQLFiles (SQLFileName, MyString) values (execute (#SqlStatement), 'aa')
or :
declare #ExecutedStmt varchar(max)
set #ExecutedStmt = execute (#SqlStatement)
-- But the above line is not correct.
Any of these 2 would work for me. I don't want to insert and update afterwards the second column.
Any ideas on how I can achieve this? Thank you !
Try using a table variable as an intermediary - not the prettiest, but does the job:
create table #tmpSQLFiles(SQLFileName varchar(max), MyString varchar(max))
set #SqlStatement = 'master.dbo.xp_cmdshell ''dir /b "' + #DirPath + '*.pdf"'''
DECLARE #TempOutpt TABLE ([output] varchar(500))
insert into #TempOutpt ([output])
execute (#SqlStatement)
insert into #tmpSQLFiles (SQLFileName, MyString)
select [output], 'my string here' from #TempOutpt
select * from #tmpSQLFiles
If you weren't using an extended stored procedure (xp_cmdshell), you might have been able to use OPENROWSET to execute the SP:
SELECT *fields*
FROM OPENROWSET('SQLNCLI',
'Server=*your_server*;Trusted_Connection=yes;Database=*your_database*',
'exec *stored_procedure* *parameters*')
You need to use an output parameter so you can use the results in the next INSERT:
DECLARE #result INT, #sql NVARCHAR(512)
SET #sql = N'SELECT #result = COUNT(*) FROM ' + #dbname + '.INFORMATION_SCHEMA.TABLES'
EXEC sp_executesql
#query = #sql,
#params = N'#result INT OUTPUT',
#result = #result OUTPUT
PRINT #result