I am exporting files to .csv using bulk copy utility, I have eight
tables and want to avoid making eight different procedures, How can I
put it all inside this one ?
Here is my attempt...
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
IF OBJECT_ID ('[dbo].[generateCSV]') IS NOT NULL
DROP PROCEDURE [dbo].[generateCSV]
GO
CREATE PROCEDURE [dbo].[generateCSV]
(
#table varchar(100),
#output varchar(100),
#date varchar(12),
#server varchar(30)
)
AS
BEGIN
DECLARE #sqlClients varchar(8000)
SELECT #sqlClients =
CASE #table WHEN 'Clients' THEN 'bcp "select
* from ' + DB_NAME() + '.dbo.' + #table + '
where ReportingDate = ''' + #date + '''"' + ' queryout ' + #output + ' -c
-C65001 -t";" -r"\n" -T -S' + #server
WHEN 'Receivables' THEN 'bcp "SELECT * from ' + DB_NAME() + + '.dbo.' +
#table + ' where ReportingDate = ''' + #date + '''"' + ' queryout ' +
#output + ' -c -C65001 -t";" -r"\n" -T -S' + #server
...
END
exec master..xp_cmdshell #sqlClients
-- Main EXEC
EXEC dbo.generateCSV #table = 'Clients', #date = '2017-10-31', #output =
'//172.18.16.109/share/Test.csv (server with export target location ) ',
#server = '172.18.16.108(server we are connected to and from which we are
taking the data)'
I have eight more tables,besides Clients and Receivables,and based on #table parameter provided in the main procedure call I want different bcp select.
How can I do this?
When I run the above script I will receive:
The error is due to inappropriate ending of case statement while assigning value to #sqlClients. Just add 'End ' keyword at the end of below statement -
SELECT #sqlClients =
CASE #table WHEN 'Clients' THEN 'bcp "select
* from ' + DB_NAME() + '.dbo.' + #table + '
where ReportingDate = ''' + #date + '''"' + ' queryout ' + #output + ' -c
-C65001 -t";" -r"\n" -T -S' + #server
WHEN 'Receivables' THEN 'bcp "SELECT * from ' + DB_NAME() + + '.dbo.' +
#table + ' where ReportingDate = ''' + #date + '''"' + ' queryout ' +
#output + ' -c -C65001 -t";" -r"\n" -T -S' + #server end --<<<----MISSING END KEYWORD
Related
I have a stored procedure begin called like this:
EXEC spExportData_n 'dbName', #SQL, #FileName
where #SQL is the SQL statement and #FileName is the file location.
It works great, however, the column headings appear randomly in the file. How can I make it so that the column headings are always the first row?
Here is the stored procedure:
CREATE PROCEDURE [dbo].[spExportData_n]
(#dbName varchar(100),
#sql varchar(8000),
#fullFileName varchar(1000))
AS
IF #sql = '' or #fullFileName = ''
BEGIN
SELECT 0 AS ReturnValue -- failure
RETURN
END
-- if DB isn't passed in set it to master
SELECT #dbName = 'use ' + #dbName + ';'
IF object_id('##TempExportData') IS NOT NULL
DROP TABLE ##TempExportData
IF object_id('##TempExportData2') IS NOT NULL
DROP TABLE ##TempExportData2
-- insert data into a global temp table
DECLARE #columnNames varchar(8000),
#columnConvert varchar(8000),
#tempSQL varchar(8000)
SELECT #tempSQL = LEFT(#sql, CHARINDEX('from', #sql)-1) + ' into ##TempExportData ' +
substring(#sql, charindex('from', #sql)-1, len(#sql))
exec(#dbName + #tempSQL)
if ##error > 0
begin
select 0 as ReturnValue -- failure
return
end
-- build 2 lists
-- 1. column names
-- 2. columns converted to nvarchar
SELECT #columnNames = COALESCE( #columnNames + ',', '') + column_name,
#columnConvert = COALESCE( #columnConvert + ',', '') + 'convert(nvarchar(4000),'
+ column_name + case when data_type in ('datetime', 'smalldatetime') then ',121'
when data_type in ('numeric', 'decimal') then ',128'
when data_type in ('float', 'real', 'money', 'smallmoney') then ',2'
when data_type in ('datetime', 'smalldatetime') then ',120'
else ''
end + ') as ' + column_name
FROM tempdb.INFORMATION_SCHEMA.Columns
WHERE table_name = '##TempExportData'
-- execute select query to insert data and column names into new temp table
SELECT #sql = 'select ' + #columnNames + ' into ##TempExportData2 from (select ' + #columnConvert + ', ''2'' as [temp##SortID] from ##TempExportData union all select ''' + replace(#columnNames, ',', ''', ''') + ''', ''1'') t order by [temp##SortID]'
exec (#sql)
-- build full BCP query
DECLARE #bcpCommand VARCHAR(8000)
SET #bcpCommand = 'bcp " SELECT * from ##TempExportData2" queryout'
SET #bcpCommand = #bcpCommand + ' ' + #fullFileName + ' -c -t, -U ***-P ****-c'
EXEC MASTER..xp_cmdshell #bcpCommand
if ##error > 0
begin
select 0 as ReturnValue -- failure
return
end
drop table ##TempExportData
drop table ##TempExportData2
set #columnNames =' '
set #columnConvert =' '
set #tempSQL =' '
select 1 as ReturnValue
I have a stored procedure that generates Insert and Update statements. So I want the proc to save the results to a .sql file.
ALTER PROCEDURE [dbo].[Sp_GenerateScripts]
#FilePath NVARCHAR(MAX) = ''
AS
BEGIN
SET NOCOUNT ON;
DECLARE #Message NVARCHAR(155)
DECLARE #Query NVARCHAR(MAX) = ''
DECLARE #GenerateInserts NVARCHAR(MAX) = ''
DECLARE #GenerateUpdates NVARCHAR(MAX) = ''
IF OBJECT_ID('tempdb..##DataImport') IS NOT NULL DROP TABLE ##DataImport
CREATE TABLE ##DataImport
(
[LongTextId] varchar(150) NULL,
[Code] [nvarchar](255) NULL,
[Product] [nvarchar](max) NULL,
[Category] [nvarchar](max) NULL,
[Subsegment] [nvarchar](max) NULL,
[DescriptionText] [nvarchar](max) NULL,
[eDate] [nvarchar](max) NULL,
[ExpirationDate] [nvarchar](max) NULL,
[UpdatedOn] [nvarchar](max) NULL
)
--BULK INSERT ##DataImport from 'C:\CSV Files\file.csv' WITH
SET #Query = 'BULK INSERT ##DataImport FROM ''' + #FilePath + ''' WITH
(
FIELDTERMINATOR = ''|'',
FIRSTROW = 2
)';
exec sp_executesql #Query;
/*CSV Data Cleansing*/
UPDATE ##DataImport
set [LongTextId] = NEWID()
WHERE LEN([LongTextId]) != 36
UPDATE ##DataImport
SET UpdatedOn = SUBSTRING(UpdatedOn, 1, CHARINDEX('|', UpdatedOn) - 1)
DELETE ##DataImport WHERE LongTextId IS NULL;
SELECT 'BEGIN TRAN'
UNION ALL
SELECT
--*,
'UPDATE [dbo].[Table1] SET'
+ ' [Code] = ''' + [Code] + ''''
+ ', [Product] = ''' + [Product] + ''''
+ ', [Category] = ''' + [Category] + ''''
+ ', [Subsegment] = ''' + [Subsegment] + ''''
+ ', [DescriptionText] = ''' + REPLACE([DescriptionText], '''', '''''') + ''''
+ ', [Date] = ''' + [EffectiveDate] + ''''
+ ', [ExpirationDate] = ''' + [ExpirationDate] + ''''
+ ', [UpdatedOn] = ''' + [UpdatedOn] + ''''
+ ' WHERE [LongTextId] = ''' + [LongTextId] + ''';' + CHAR(10)
from ##DataImport src
WHERE EXISTS
(
SELECT 1
FROM
[dbo].TABLE1 AS dest
WHERE
dest.[LongTextId] = src.[LongTextId]
)
UNION ALL
SELECT 'INSERT INTO [dbo].[TABLE1] ([Subsegment], [DescriptionText], [LongTextId], [Code], [Date], [ExpirationDate], [Product], [Category], [UpdatedOn]) VALUES ('
+ '''' + [CustomerSubsegment] + ''', '
+ '''' + REPLACE([DescriptionText], '''', '''''') + ''', '
+ '''' + [LongTextId] + ''', '
+ '''' + [Code] + ''', '
+ '''' + [Date] + ''', '
+ '''' + [Expirationdate] + ''', '
+ '''' + [Product] + ''', '
+ '''' + [Category] + ''', '
+ '''' + [UpdatedOn] + ''');'
from ##DataImport src
WHERE NOT EXISTS
(
SELECT 1
FROM
[dbo].[TABLE1] AS dest
WHERE
dest.[LongTextId] = src.[LongTextId]
)
UNION ALL
SELECT 'COMMIT TRAN'
END
This proc works fine and this is how I call it:
EXEC [dbo].[Sp_GenerateScripts] 'C:\CSV Files\Text_3.csv'
What I need to do is to write the results of the stored proc to a .sql file somewhere to a shared folder or local folder doesn't matter. So I have tried to do it in powershell as this
sqlcmd -S SERVER-PC -d DB1 -E -Q "EXEC [dbo].[Sp_GenerateScripts] 'C:\CSV Files\Text_3.csv'" -o "C:\Export.sql"
In powershell its working but I don't want to use this method. Is there a way I can do this in T-SQL? If yes, how can I modify my stored proc to do this?
You can execute the same sqlcmd command from TSQL using xp_Cmdshell. But, using xp_cmdshell comes with its own risks, as it can run system commands. So, care should be taken care before enabling this option using sp_configure. Also, if you are planning to use this, it is better to execute using least privileged user context. You can configure proxy account to execute with xp_cmdshell using sp_xp_cmdshell_proxy_account.
Read more on xp_cmdshell
EXEC xp_cmdshell 'sqlcmd -S SERVER-PC -d DB1 -E
-Q "EXEC [dbo].[Sp_GenerateScripts] ''C:\CSV Files\Text_3.csv''" -o "C:\Export.sql"'
If xp_cmdshell is not recommended as a security practice, alternatively, you can have a CLR stored procedure, especially for this purpose. Again, that requires enabling CLR using sp_configure.
How can I ignore null's in my exported .csv file, I used bcp utility
within sql server, here it is how this looks like altogether.
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
IF OBJECT_ID ('[dbo].[generateCSV]') IS NOT NULL
DROP PROCEDURE [dbo].[generateCSV]
GO
CREATE PROCEDURE [dbo].[generateCSV]
(
#table varchar(100),
#output varchar(100),
#date varchar(12),
#server varchar(30)
)
AS
DECLARE #sql varchar(8000)
SELECT #sql = 'bcp "select * from ' + DB_NAME() + '.dbo.' + #table + '
where reportingdate = ''' + #date + '''"' + ' queryout ' + #output + ' -c
-C65001 -t";" -r"\n" -T -S' + #server
exec master..xp_cmdshell #sql
-- Main EXEC
EXEC dbo.generateCSV #table = 'Clients', #date = '2017-10-31', #output =
'//172.18.16.109/share/Test.csv (server with export target location ) ',
#server = '172.18.16.108(server we are connected to and from which we are
taking the data)'
After exporting and opening my file in notepad++ empty columns end up filled with NULL's.
What I want it to look like is...
Edit:
I included ISNULL() in my select clause but still notepad will read those columns as null valued.
SELECT #sql = 'bcp "select
ReportingDate,uniqClientID,registrationNumber,name,ISNULL(vatNumber,'''') as
vatNumber,ISNULL(entityStatusCode,'''') as entityStatusCode,
maximumLifetimeDPD from ' + DB_NAME() + '.dbo.' +
#table + ' where ReportingDate = ''' + #date + '''"' + ' queryout ' +
#output + ' -c -C65001 -t";" -r"\n" -T -S' + #server
SELECT #sql = 'bcp "select * from ' + DB_NAME() + '.dbo.' + #table +
' where reportingdate = ''' + #date + '''"' + ' queryout ' + #output
+ ' -c
-C65001 -t";" -r"\n" -T -S' + #server
Instead of SELECT * write the specific column names.
Wherever you feel NULL can appear you can write like following.
....
SELECT
Column1,
ISNULL(Column2,'') AS Column2,
Column3,
....
from ' + DB_NAME() + '.dbo.' + #table
.....
Note: If you have INT as datatype, you need to slightly change your select according to following query.
DECLARE #INTVAL INT
SET #INTVAL = NULL
SELECT
ISNULL(LTRIM(NULLIF(#INTVAL, 0)), '') AS ColumnName
I'm having issues with SQL Server's BCP. I'm not getting an error message but the command is also not writing anything to the path specified.
Here is my code:
In this scenario, the only variable not accounted for in the code below is #MESSAGE which is a NVARCHAR(MAX) and will take up most of that space so VARCHAR(8000) won't be nearly big enough.
DECLARE #OUTPUT_TABLE VARCHAR(255) = '##temp' + CONVERT(VARCHAR(12), CONVERT(INT, RAND() * 1000000))
DECLARE #RESULT INTEGER
DECLARE #OUTPUT_EXECUTION_COMMAND NVARCHAR(MAX) = 'CREATE TABLE [' + #OUTPUT_TABLE + '] ( CONTENT NVARCHAR(MAX) )
INSERT INTO [' + #OUTPUT_TABLE + ']
SELECT #MESSAGE'
EXECUTE SP_EXECUTESQL #OUTPUT_EXECUTION_COMMAND, N'#MESSAGE NVARCHAR(MAX)', #MESSAGE
DECLARE #OUTPUT_FILE_WRITE_COMMAND VARCHAR(1000) = 'BCP ”SELECT CONTENT FROM [' + #OUTPUT_TABLE + ']"
QUERYOUT "' + #FULL_PATH + '\' + #FILE_NAME + '" -w -T -S ' + ##servername
PRINT #OUTPUT_FILE_WRITE_COMMAND
EXECUTE #RESULT = master.sys.xp_cmdshell #OUTPUT_FILE_WRITE_COMMAND, NO_OUTPUT
PRINT #RESULT
EXECUTE ( 'DROP TABLE [' + #OUTPUT_TABLE + ']' )
So, the frustrating part is that I'm receiving an error, yet my file is not being written.
The output of PRINT #OUTPUT_FILE_WRITE_COMMAND is:
BCP ”SELECT CONTENT FROM [##temp878274]"
QUERYOUT "\\TXPDC-FS01\Profiles\cofarmer\My Sandbox\THCIC\Q2_2014\Burleson\PIPSUB2938718184092014251607.txt" -w -T -S TXPDC-STKSQL01
While the output of PRINT #RESULT is: 1
Yet nothing is being written. What am I doing wrong?
Looks like the issue is the line break in the#OUTPUT_FILE_WRITE_COMMAND (and the erroneous double quote”). Removing the line break worked for me:
DECLARE #OUTPUT_FILE_WRITE_COMMAND VARCHAR(1000) = 'BCP "SELECT CONTENT FROM [' + #OUTPUT_TABLE + ']" QUERYOUT "' + #FULL_PATH + '\' + #FILE_NAME + '" -w -T -S ' + ##servername
Result: 0 and the file was created.
I'm building a stored procedure to transfer data from SQL Server to a remote FTP server.
I wrote the table data to a file with the following query
exec xp_cmdshell "bcp master.dbo.table out c:\temp\bcp_outputFile.txt -S(local)\SQLEXPRESS -T -c"
However, I'm struggling with building a script that would take that file and send it across to the FTP Server.
Using SSIS is not an option. It must be done with the xp_cmdshell command.
Any ideas?
Thanks
Use the below mention procedure for uploading the file in the ftp server.
here in the procedure i have hard coded the ftp details you can change it.
create procedure [dbo].[uploadfile_ftp]
#SourcePath varchar(128),
#SourceFiles varchar(128),
#DestPath varchar(128)
AS
Begin
DECLARE #FTPServer varchar(128)
DECLARE #FTPUser varchar(128)
DECLARE #FTPPwd varchar(128)
DECLARE #FTPMode varchar(10)
-- FTP attributes.
SET #FTPServer = 'ftp url'
SET #FTPUser = 'ftp username'
SET #FTPPwd = 'ftp password'
SET #FTPMode = 'binary' -- ascii, binary or blank for default.
DECLARE #cmd varchar(1000)
DECLARE #workfile varchar(128)
DECLARE #nowstr varchar(25)
-- Get the %TEMP% environment variable.
DECLARE #tempdir varchar(128)
CREATE TABLE #tempvartable(info VARCHAR(1000))
INSERT #tempvartable EXEC master..xp_cmdshell 'echo %temp%'
SET #tempdir = (SELECT top 1 info FROM #tempvartable)
IF RIGHT(#tempdir, 1) <> '\' SET #tempdir = #tempdir + '\'
DROP TABLE #tempvartable
-- Generate #workfile
SET #nowstr = replace(replace(convert(varchar(30), GETDATE(), 121), ' ', '_'), ':', '-')
SET #workfile = 'FTP_SPID' + convert(varchar(128), ##spid) + '_' + #nowstr + '.txt'
-- Deal with special chars for echo commands.
select #FTPServer = replace(replace(replace(#FTPServer, '|', '^|'),'<','^<'),'>','^>')
select #FTPUser = replace(replace(replace(#FTPUser, '|', '^|'),'<','^<'),'>','^>')
select #FTPPwd = replace(replace(replace(#FTPPwd, '&', '^&'),'<','^<'),'>','^>')
select #DestPath = replace(replace(replace(#DestPath, '|', '^|'),'<','^<'),'>','^>')
IF RIGHT(#SourcePath, 1) <> '\' SET #SourcePath = #SourcePath + '\'
-- Build the FTP script file.
select #cmd = 'echo ' + 'open ' + #FTPServer + ' > ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
print #cmd;
select #cmd = 'echo ' + #FTPUser + '>> ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
print #cmd;
select #cmd = 'echo ' + #FTPPwd + '>> ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
print #cmd;
select #cmd = 'echo ' + 'prompt ' + ' >> ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
print #cmd;
IF LEN(#FTPMode) > 0
BEGIN
select #cmd = 'echo ' + #FTPMode + ' >> ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
END
IF LEN(#DestPath) > 0
BEGIN
select #cmd = 'echo ' + 'cd ' + #DestPath + ' >> ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
END
select #cmd = 'echo ' + 'mput ' + #SourcePath + #SourceFiles + ' >> ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
select #cmd = 'echo ' + 'quit' + ' >> ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
-- Execute the FTP command via script file.
select #cmd = 'ftp -s:' + #tempdir + #workfile
print #cmd;
create table #a (id int identity(1,1), s varchar(1000))
insert #a
EXEC master..xp_cmdshell #cmd
select id, ouputtmp = s from #a
-- Clean up.
drop table #a
select #cmd = 'del ' + #tempdir + #workfile
EXEC master..xp_cmdshell #cmd
End