Crawler html site with SQL Server (Ole Automation Procedures) - sql-server

I want to save HTML sites in a table with this query - but it returns Null:
DECLARE #ResponseTXT NVARCHAR(MAX)
DECLARE #obj INT
DECLARE #ValorDeRegreso INT
DECLARE #sUrl NVARCHAR(200)
DECLARE #hr INT
DECLARE #src NVARCHAR(255)
DECLARE #desc NVARCHAR(255)
DECLARE #StringRequest NVARCHAR(500)=''
SET #sUrl = 'http://www.myfootballgames.co.uk' -- as an example
EXEC sp_OACreate 'MSXML2.ServerXMLHttp', #obj OUT
EXEC sp_OAMethod #obj, 'Open', NULL, 'GET', #sUrl, false
EXEC sp_OAMethod #obj, 'setRequestHeader', NULL, 'Content-Type', 'text/html; charset=utf-8'--'application/x-www-form-urlencoded'
EXEC sp_OAMethod #obj, 'send', NULL, #StringRequest
EXEC sp_OAGetProperty #obj, 'responseText', #ResponseTXT OUT
SELECT #ResponseTXT

Related

SOAP 1.1 request from t-sql

I have copied the code from setting soap request properties from T-sql and make some litle changes.
If soap version 1.2 the code works fine. My problem is that soap version 1.1 wants to load the PELTT01.wsdl file to work.
declare
#Url varchar(1024),
#HttpMethod varchar(10),
#ParamsValues varchar(1024),
#SoapAction varchar(8000),
#datefrom varchar(30),
#dateto varchar(30),
#sessionid as varchar(3000),
#xmlOut1 varchar(8000),
#RequestText as varchar(8000),
#wsdl as varchar(8000),
#responseHeader varchar(4000)
DECLARE #t table (ID int, strxml xml)
set #Url = 'http://212.205.47.226:9003'
set #HttpMethod = 'soap'
set #SoapAction = '<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:pel="/PELTT01">
<soapenv:Header/>
<soapenv:Body>
<pel:READ>
<pel:wpel_code>999999999</pel:wpel_code>
<pel:wpel_user>9999999</pel:wpel_user>
<pel:wpel_pass>9999999</pel:wpel_pass>
<pel:wpel_vg>ZK000000000GR</pel:wpel_vg>
<pel:wpel_ref>?</pel:wpel_ref>
<pel:wpel_flag>1</pel:wpel_flag>
</pel:READ>
</soapenv:Body>
</soapenv:Envelope>'
declare #obj int
,#response varchar(8000)
,#responseText varchar(8000)
,#status varchar(50)
,#statusText varchar(1024)
,#method varchar(10) = (case when #HttpMethod in ('soap','SOAP') then 'POST' else #HttpMethod end)
exec sp_OACreate 'MSXML2.ServerXMLHTTP', #obj out
exec sp_OAMethod #obj, 'Open', null, #method, #Url, 'false'
exec sp_OAMethod #obj, 'setRequestHeader', null, 'Content-Type', 'text/xml; charset=utf-8'
exec sp_OAMethod #obj, 'setRequestHeader', null, 'SOAPAction', #SoapAction
exec sp_OAMethod #obj, 'send', null, #SoapAction
exec sp_OAGetProperty #obj, 'responseText', #responseText out
EXECUTE sp_OAMethod #Obj, 'getAllResponseHeaders', #responseHeader OUT
exec sp_OAGetProperty #obj, 'Status', #status out
exec sp_OADestroy #obj
SELECT #status 'status', #responseText 'responsetext', #responseHeader 'Response Header'

Get SOAP web service gzip enconding response from SQL Server

I'm trying to call a SOAP web service from SQL Server using OLE Automation stored procedures, and I've got troubles with the response which is Content-Encoding gzip.
When test the method using SOAP UI works like a charm.
Soap UI shows response like xml. Http log shows response is gzip encoded
When I make the call from SQL Server, I get an unexpected response like this
XML response from SQL Server
Status return is ok 200.
GetAllResponseHeaders doesn't include 'Content-Encoding: gzip', Soap UI does!
I included all request headers showed on SOAP UI.
Here is my code (I omitted error handling for the sake of readability)
DECLARE
#URI varchar(2000) = 'http://foo/foo.wsdl',
#methodName varchar(50) = 'post',
#requestBody varchar(8000) = '<soapenv:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:urn="urn:CCTwebservices"><soapenv:Header/><soapenv:Body><urn:recepcionLiberacionTatc soapenv:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"><user xsi:type="xsd:string">foo</user><password xsi:type="xsd:string">foo</password><dataIn xsi:type="xsd:string"><![CDATA[foo]]></dataIn></urn:recepcionLiberacionTatc></soapenv:Body></soapenv:Envelope>',
#SoapAction varchar(255) = 'urn:CCTwebservices#recepcionLiberacionTatc',
#responseText varchar(8000);
DECLARE #objectID int
DECLARE #hResult int
DECLARE #source varchar(255), #desc varchar(255)
--create object
EXEC #hResult = sp_OACreate 'MSXML2.ServerXMLHTTP', #objectID OUT
-- open the destination URI with Specified method
EXEC #hResult = sp_OAMethod #objectID, 'open', null, #methodName, #URI, 'false', null, null
-- set request headers
EXEC #hResult = sp_OAMethod #objectID, 'setRequestHeader', null, 'Content-Type', 'text/xml;charset=UTF-8'
EXEC #hResult = sp_OAMethod #objectID, 'setRequestHeader', null, 'Accept-Encoding', 'gzip,deflate'
EXEC #hResult = sp_OAMethod #objectID, 'setRequestHeader', null, 'Host', 'ws.foo`enter code here`.cl'
EXEC #hResult = sp_OAMethod #objectID, 'setRequestHeader', null, 'Connection', 'Keep-Alive'
EXEC #hResult = sp_OAMethod #objectID, 'setRequestHeader', null, 'SOAPAction', #SoapAction
declare #len int
set #len = len(#requestBody)
EXEC #hResult = sp_OAMethod #objectID, 'setRequestHeader', null, 'Content-Length', #len
-- send the request
EXEC #hResult = sp_OAMethod #objectID, 'send', null, #requestBody
EXEC #hResult = sp_OAMethod #objectID, 'getResponseHeader', null, 'Content-Encoding' -- Return null
EXEC #hResult = sp_OAMethod #objectID, 'GetAllResponseHeaders' -- Response doesn't incluye 'Content-Encoding: gzip', Soap UI does!.
exec sp_OAGetProperty #objectID, 'StatusText'
exec sp_OAGetProperty #objectID, 'Status'
exec sp_OAGetProperty #objectID, 'responseText' -- return the xml show previosly
exec sp_OADestroy #objectID
Any help to make this work?

Calling API from SQL Server [Issue]

I am working with API and MSSQL. I need to call an API from SQL Server. But On execution, its response is null. Can anyone list errors if some in the queries?
The API responds but the issue is it shows error in Authentication while I'm using exact correct credentials. It's a POST API.
This is the only way I found correct on browsing the whole a day on Google. Can someone help or list errors?
DECLARE #authHeader NVARCHAR(MAX)= 'Basic XXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXXXXXXXXXXXX'
DECLARE #contentType NVARCHAR(64) = 'application/json; charset=utf-8'
DECLARE #appVersion NVARCHAR(20) = '3.2.1'
DECLARE #appkey NVARCHAR(50) = 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
DECLARE #deviceId NVARCHAR(50) = 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
DECLARE #deviceType int = 1;
DECLARE #language int = 1;
DECLARE #status NVARCHAR(32);
DECLARE #statusText NVARCHAR(32);
DECLARE #Body as VARCHAR(8000) =
'{
"itemId":"XXXXXXXXXXXXXXXXXXXXXXX",
"itemType":XXXXXXXXXXXXX,
"liked":"true"
}'
DECLARE #responseText NVARCHAR(2000);
DECLARE #token INT;
DECLARE #url NVARCHAR(256) = 'https://api-stage.testapp.com/v4/test';
Exec sp_OACreate 'MSXML2.ServerXMLHTTP', #token OUT;
EXEC sp_OAMethod #token, 'open', NULL, 'POST', #url, 'false'
EXEC sp_OAMethod #token, 'setRequestHeader', NULL, 'Authentication', #authHeader
EXEC sp_OAMethod #token, 'setRequestHeader', NULL, 'Content-type', #contentType
EXEC sp_OAMethod #token, 'setRequestHeader', NULL, 'appVersion', #appVersion
EXEC sp_OAMethod #token, 'setRequestHeader', NULL, 'appkey', #appkey
EXEC sp_OAMethod #token, 'setRequestHeader', NULL, 'deviceId', #deviceId
EXEC sp_OAMethod #token, 'setRequestHeader', NULL, 'devicetype', #deviceType
EXEC sp_OAMethod #token, 'setRequestHeader', NULL, 'language', #language
EXEC sp_OAMethod #token, 'send', null
Exec sp_OAMethod #token, 'setRequestBody', null, 'Body', #body
EXEC sp_OAGetProperty #token, 'status', #status OUT;
EXEC sp_OAGetProperty #token, 'statusText', #statusText OUT;
EXEC sp_OAGetProperty #token, 'responseText', #responseText OUT;
PRINT 'Status: ' + #status + ' (' + #statusText + ')';
PRINT 'Response text: ' + #responseText;
Select #ResponseText as Response

SQL Server - Call a REST API using a virtual CSV

I have the code below that makes a request to an API and works fine, using JSON content-type to another endpoint.
DECLARE #Object AS int;
DECLARE #ResponSEText AS Varchar(8000);
DECLARE #Token AS Varchar(8000);
DECLARE #xmltest AS Varchar(8000);
DECLARE #hResult AS int
DECLARE #source varchar(255), #desc varchar(255)
DECLARE #LocationId varchar(25);
DECLARE #Body AS varchar(8000) = '{
"UserName": "test.integration12",
"Password": "Urgent123"
}'
--EXEC sp_OACREATE 'MSXML2.XMLHTTP', #Object OUT;
EXEC sp_OACREATE 'MSXML2.ServerXMLHttp', #Object OUT;
EXEC sp_OAMethod #Object, 'open', NULL, 'POST',
'https://urgentcargus.azure-api.net/api/LoginUser', 'false'
EXEC sp_OAMethod #Object, 'SETRequestHeader', null, 'Content-Type', 'application/json'
DECLARE #len int
SET #len = len(#body)
EXEC sp_OAMethod #Object, 'SETRequestHeader', null, 'Ocp-Apim-Subscription-Key','4f82f9d067914287979884f920d86ffb'
EXEC sp_OAMethod #Object, 'SETRequestHeader', null, 'Ocp-Apim-Trace:true'
EXEC sp_OAMethod #Object, 'SETRequestHeader', null, 'Content-Length', #len
EXEC sp_OAMethod #Object, 'SETRequestBody', null, 'Body', #body
EXEC sp_OAMethod #Object, 'sEND', null, #body
EXEC sp_OAMethod #Object, 'responSEText', #ResponSEText OUTPUT
I would like to adapt the code above in order to send the values of the parameters like a virtual csv file instead of an XML or JSON protocol with the appropriate format to another endpoint using a different content-type and I cannot figure it out how to do this:
DECLARE #Object AS int;
DECLARE #ResponSEText AS Varchar(8000);
DECLARE #Token AS Varchar(8000);
DECLARE #xmltest AS Varchar(8000);
DECLARE #hResult AS int
DECLARE #source varchar(255), #desc varchar(255)
DECLARE #LocationId varchar(25);
DECLARE #Body AS varchar(8000) = -- here should go the parameters which are username, clientid, password and file (where file has to be the virtual CSV file with the columns and rows separated by "|"
EXEC sp_OACREATE 'MSXML2.ServerXMLHttp', #Object OUT;
EXEC sp_OAMethod #Object, 'open', NULL, 'POST',
'https://www.selfawb.ro/import_awb_integrat.php', 'false'
EXEC sp_OAMethod #Object, 'SETRequestHeader', null, 'Content-Type', 'text/xml'
EXEC sp_OAMethod #Object, 'SETRequestHeader', null, 'Ocp-Apim-Trace:true'
EXEC sp_OAMethod #Object, 'SETRequestBody', null, 'Body', #body
EXEC sp_OAMethod #Object, 'sEND', null, #body
EXEC sp_OAMethod #Object, 'responSEText', #ResponSEText OUTPUT
I asked the owner of the API and they provided to me an example as shown in the print screen below:
Example
Does anyone can give me a hint or a hand? Thanks
If you want to send a virtual file, e.g. a file that you generate in memory then send as an attachment as part of the POST, then you would just need include the csv text as part of the post as an attachment with a filename. Try using PostMan to set this up using an actual csv file and analyze the POST to view how to format the csv text to be serialized over the wire as a csv file attachment.
Otherwise if you are using SQL 2016 or above, you can parse csv text using STRING_SPLIT. Hope this helps.
DECLARE #CsvParams NVARCHAR(256) =
'UserName,test.integration12,'
+ 'Password,Urgent123'
SELECT value FROM STRING_SPLIT(#CsvParams, ',');
The code above produces a table with the separated values:
More on STRING_SPLIT here:
STRING_SPLIT (Transact-SQL)

Consuming webservice by SQL Server procedure

I am using sql server 2016 to consume webservice through store procedure. I have a webservice in JSON returning the following:
["Name":"Rebecca","email":"rebecca#hotmail.com","ra":"12345"},
{"name":"Caroline","email":"caroline#hotmail.com","ra":"23456"},
{"name":"Vanessa","email":"vanessa#yahoo.com.br","ra":"99999"}]
I can consume it by passing a variable. If I leave my webservice to receive a parameter and pass it I can return the name and email of the student. But if I leave my webservice without the need to receive parameter and remove the procedure parameter pass and execute, my procedure returns null. What do I need to change in my code to be able to return the complete list of students my webservice exposes?
Here's my code:
create PROCEDURE webservice_parametros (#RA as varchar (5))
AS
BEGIN
DECLARE #OBJ INT;
DECLARE #URL VARCHAR(200);
DECLARE #RESPONSE VARCHAR(8000);
SET #URL = 'http://dominio:8080/v1/alunos/' + #RA
EXEC SP_OACREATE 'MSXML2.ServerXMLHttp', #OBJ out
EXEC SP_OAMETHOD #OBJ, 'OPEN', NULL, 'GET', #URL, FALSE
EXEC SP_OAMETHOD #OBJ, 'SEND'
exec SP_OAGETPROPERTY #OBJ, 'responseText', #RESPONSE out
EXEC SP_OADESTROY #OBJ
SELECT JSON_VALUE(#RESPONSE, '$.nome') as nome,
JSON_VALUE(#RESPONSE, '$.email') as email
END
execute webservice_parametros '12345'
My code that returns NULL
create PROCEDURE webservice
AS
BEGIN
DECLARE #OBJ INT;
DECLARE #URL VARCHAR(200);
DECLARE #RESPONSE VARCHAR(8000);
SET #URL = 'http://dominio:8080/v1/alunos/'
EXEC SP_OACREATE 'MSXML2.ServerXMLHttp', #OBJ out
EXEC SP_OAMETHOD #OBJ, 'OPEN', NULL, 'GET', #URL, FALSE
EXEC SP_OAMETHOD #OBJ, 'SEND'
exec SP_OAGETPROPERTY #OBJ, 'responseText', #RESPONSE out
EXEC SP_OADESTROY #OBJ
SELECT JSON_VALUE(#RESPONSE, '$.nome') as nome,
JSON_VALUE(#RESPONSE, '$.email') as email
END
execute webservice

Resources