Node + SQL Server : Get row data as stream - sql-server

I try to use Node.Js connector with Microsoft driver to communicate with a SQL Server. In the connector docs I've found a good option named 'stream'. It add ability to asynchronously obtain row objects.
My data have some specific - some columns have large binary data (> 100 Mb). So even one row may be really large. I'm looking for ability to get each row data as a stream. It is possible in .NET driver (CommandBehavior.SequentialAccess enumeration). Is it possible in Node.js?
UPDATED
Here is a code to demonstrate the problem:
Custom writable stream module:
var stream = require('stream');
var util = require('util');
function WritableObjects() {
stream.Writable.call(
this,
{
objectMode: true
}
);
}
util.inherits( WritableObjects, stream.Writable );
WritableObjects.prototype._write = function( chunk, encoding, doneWriting ) {
console.log('write', chunk, encoding);
doneWriting();
};
module.exports = {
WritableObjects: WritableObjects
};
and database query code:
var sw = new wstream.WritableObjects();
var request = new sql.Request(connection);
request.stream = true;
request.pipe(sw);
request.query('SELECT DataId, Data FROM ds.tData WHERE DataId in (1)');
sw.on('error', function(err) {
console.log('Stream err ', err)
});
sw.on('pipe', function(src) {
console.log('Stream pipe ')
});
sw.on('finish', function(data) {
console.log('Stream finish')
});
I this example chunk parameter of _write method contains the whole data of db record, not a stream. Because Data field contains a big varbinary data memory of node process also grows huge.

Yes you can stream query with the node-mssql package as stated here: https://github.com/patriksimek/node-mssql
stream - Stream recordsets/rows instead of returning them all at once as an argument of callback (default: false). You can also enable streaming for each request independently (request.stream = true). Always set to true if you plan to work with large amount of rows.

Related

Loading large CSV file with Papaparse not working (only first chunk loaded)

I would like to load a local file (client-side) with papaparse into my React application. Unfortunately, it only loads the first chunk but never the whole file. My file contains about 500 rows and there are never more than 300 rows loaded. It seems like the complete functions is already called after the first chunk.
Since I need to navigate to another page when the file is loaded completely, this is bothering me since I need the complete file for further functions.
The code I use at the moment:
async getData() {
const self = this;
let dataList = [];
Papa.parse(await this.fetchCsv(),
{
delimiter: ',',
header: true,
chunk: function (result, parser) {
parser.pause();
dataList = dataList.concat(result.data)
parser.resume();
},
complete: function () {
self.updateData(dataList);
}
});
}
async fetchCsv() {
const response = await fetch(this.props.location.state.filename);
const reader = response.body.getReader();
const result = await reader.read();
const decoder = new TextDecoder('utf-8');
return decoder.decode(result.value);
}
What I've also tried is using step instead of chunk but this did not change anything.
Can anyone tell me what I'm doing wrong here and why papaparse does not load the whole file?
You may be able to let papaparse do more. It can read local File or stream data from a remote server.
If you only have about 500 records, you may not need to add the complexity associated with streaming. This is especially true if you're just accumulating the data (which it appears you are). Use streaming primarily to process the data 1 record at a time.
If you want to stream, I'd recommend using the "step" callback instead of the "chunk" callback so you can process each row of data.
If you use the step or chunk callbacks, then you don't need the complete callback. If it's called, it won't have the data.

Retrieve multiple result sets in sails js

I am using sails js with it sails-mssqlserver adapter. The problem with it is that if my stored procedure returns multiple result sets then I only receive one result set which is the latest of all.
The same stored procedure is working fine with Java and I get to iterate over the relevant result sets.
I need to know if there is some specific way to access all result sets in sails-mssqlserver?
The sails-mssqlserver adapter is a wrapper of the official Microsoft SQL Server client for Node.js available here its dependecy however is not on the latest release.
Option 1:
As per this official documentation of the MsSQL package, you can enable multiple recordsets in queries with the request.multiple = true command.
To enable multiple queries/recordsets in the sails-mssqlserver adapter, a hackish workaround is to open sails-mssqlserver/lib/adapter.js and edit the raw query function. Adding request.multiple = true below var request = new mssql.Request(mssqlConnect). As shown in the example below.
// Raw Query Interface
query: function (connection, collection, query, data, cb) {
if (_.isFunction(data)) {
if (debugging) {
console.log('Data is function. A cb was passed back')
}
cb = data
data = null
}
adapter.connectConnection(connection, function __FIND__ (err, uniqId) {
if (err) {
console.error('Error inside query __FIND__', err)
return cb(err)
}
uniqId = uniqId || false
var mssqlConnect
if (!uniqId) {
mssqlConnect = connections[connection].mssqlConnection
} else {
mssqlConnect = connections[connection].mssqlConnection[uniqId]
}
var request = new mssql.Request(mssqlConnect)
// Add it here
request.multiple = true
request.query(query, function (err, recordset) {
if (err) return cb(err)
if (connections[connection] && !connections[connection].persistent) {
mssqlConnect && mssqlConnect.close()
}
cb(null, recordset)
})
})
},
Now the returned recordset should contain multiple results.
Option 2:
A more sustainable option for use cases where running a stored procedure which returns multiple recordsets, is to use the latest version of the official Microsoft SQL Server client for Node.js. Information on running stored procedures is available here
First install the latest package:
npm install mssql --save
In your code where you would like to run the stored procedure add a connection to the mssql database:
// require the mssql package
const sql = require('mssql')
// make a connection, you can use the values you have already stored in your adapter
const pool = new sql.ConnectionPool({
user: sails.config.connections.<yourMsSQLConnection>.user,
password: sails.config.connections.<yourMsSQLConnection>.password,
server: sails.config.connections.<yourMsSQLConnection>.server,
database: sails.config.connections.<yourMsSQLConnection>.database
})
// connect the pool and test for error
pool.connect(err => {
// ...
})
// run the stored procedure using request
const request = new sql.Request()
request.execute('procedure_name', (err, result) => {
// ... error checks
console.log(result.recordsets.length) // count of recordsets returned by the procedure
console.log(result.recordsets[0].length) // count of rows contained in first recordset
console.log(result.recordset) // first recordset from result.recordsets
console.log(result.returnValue) // procedure return value
console.log(result.output) // key/value collection of output values
console.log(result.rowsAffected) // array of numbers, each number represents the number of rows affected by executed statemens
// ...
})
// you can close the pool using
pool.close()
In cases, where the sails-* database adapter doesn't include all the functionality you require. I find it best to create a sails Service that wraps the additional functionality. It is a really clean solution.

AngularJS GET receives empty reply in Chrome but not in Fiddler

I'm implementing file download using AngularJS and WCF. My back-end is a .NET project hosted in IIS. The file is serialized as an array of bytes and then on the client side I utilize the File API to save the content.
To simplify the problem, back-end is like:
[WebInvoke(Method = "GET", UriTemplate = "FileService?path={path}")]
[OperationContract]
public byte[] DownloadFileBaseOnPath(string path)
{
using (var memoryStream = new MemoryStream())
{
var fileStream = File.OpenRead(path);
fileStream.CopyTo(memoryStream);
fileStream.Close();
WebOperationContext.Current.OutgoingResponse.Headers["Content-Disposition"] = "attachment; filename=\"Whatever\"";
WebOperationContext.Current.OutgoingResponse.ContentType = "application/octet-stream"; // treat all files as binary file
return memoryStream.ToArray();
}
}
And on client side, it just sends a GET request to get those bytes, converts in into a blob and save it.
function sendGetReq(url, config) {
return $http.get(url, config).then(function(response) {
return response.data;
});
}
Save the file then:
function SaveFile(url) {
var downloadRequest = sendGetReq(url);
downloadRequest.then(function(data){
var aLink = document.createElement('a');
var byteArray = new Uint8Array(data);
var blob = new Blob([byteArray], { type: 'application/octet-stream'});
var downloadUrl = URL.createObjectURL(blob);
aLink.setAttribute('href', downloadUrl);
aLink.setAttribute('download', fileNameDoesNotMatter);
if (document.createEvent) {
var event = document.createEvent('MouseEvents');
event.initEvent('click', false, false);
aLink.dispatchEvent(event);
}
else {
aLink.click();
}
setTimeout(function () {
URL.revokeObjectURL(downloadUrl);
}, 1000); // cleanup
});
}
This approach works fine with small files. I could successfully download files up to 64MB. But when I try to download a file larger than 64MB, the response.body is empty in Chrome. I also used Fiddler to capture the traffic. According to Fiddler, Back-end has successfully serialized the byte array and returned it. Please refer to the screenshot below.
In this example, I was trying to download a 70MB file:
And the response.data is empty:
Any idea why this is empty for file over 70MB? Though the response itself is more than 200MB, I do have enough memory for that.
Regarding to the WCF back-end, I know I should use Stream Mode when it comes to large files. But the typical use of my application is to download files less than 10MB. So I hope to figure this out first.
Thanks
Answer my own question.
Honestly I don't know what's going wrong. The issue still persists if I transfer it as a byte array. I eventually gave up this approach by returning a stream instead. Then on the client side, adding the following configuration
{responseType : blob}
and save it as a blob.

Can't retrieve more then 50 records from Azure MobileServiceClient (Node.js)

So I'm building an Apache Cordova app (Android), angularjs based, and using the WindowsAzure MobileServiceClient library to retrieve data from my Easy Table's, created in a SQL Database.
This works! Until I'd like to retrieve more then 50 records.
So I added the .take(100) to my table-read. Still 50 records..
Then I thought, maybe the take function doesn't work at all, so I changed the amount to 5, and I only got 5 records. So the take function is somewhat working, but not above 50 records.
Since it's a node.js backend, how do I increase the pagesize?
Here's my current code:
msc.tables.pokemonType = null;
msc.tables.pokemon = null;
msc.init = function () {
console.log('MobileServiceClient initializing...');
var mobileServiceClient = new WindowsAzure.MobileServiceClient("https://blablablabla");
msc.tables.pokemonType = mobileServiceClient.getTable('PokemonType');
msc.tables.pokemon = mobileServiceClient.getTable('Pokemon');
console.log('MobileServiceClient initialized!');
}
msc.getAllPokemonTypes = function() {
console.log('getAllPokemonTypes - called');
return msc.tables.pokemonType.read().then(function (pokemonTypes) {
console.log('getAllPokemonTypes - done');
return pokemonTypes;
}, msc.handleError);
}
msc.getAllPokemons = function () {
console.log('getAllPokemons - called');
return msc.tables.pokemon.take(100).read().then(function (pokemons) {
console.log('getAllPokemons - done');
return pokemons;
}, msc.handleError);
}
According the source code of table operations of Mobile Apps in node.js, the read operation ultimately receives context.query which is a queryjs object, which contains a take() function which can limit the number of items returned to the specified number.
Additionally, the take() function is contained in the mobile app server sdk, so it doesn't work on your client end code.
You can do some modification on your Easy Tables scripts, E.G.
table.read(function (context) {
context.query.take(100);
return context.execute();
});

Is it possible to save a file directly from a web worker?

I have an entirely browser-based (i.e. no backend) application which analyzes XML data in files which average about 250MB each. The actual parsing and analysis happens in a web worker, which is fed data in 64KB chunks by a FileReader instance, and this is all quite performant.
I have a request from the client to expand this application so that it can generate a .zip file containing the original input file and the results of the analysis, and allow the user to save that file to her local machine. Generating a .zip file in memory with those contents isn't a problem. The problem lies in transferring that much data from the web worker which generates it back to the main browser thread, so that it can be saved; attempting to do this invariably provokes a crash or out-of-memory exception. (I've tried transferring strings all at once and a chunk at a time, and I've tried using an ArrayBuffer as a transferable object to avoid copying. All fail in the same fashion.)
Unfortunately, I don't know any way to invoke a file save operation directly from a worker thread. I know several methods of doing so from the main browser thread, but all of them require either the ability to create DOM nodes (which worker threads of course can't do), or the use of interfaces (i.e. msSaveBlob, saveAs) which no browser seems to expose to a worker thread. I've spent a while looking for possibilities on the web, but found nothing usable; FileWriterSync looked good, but only Chrome supports it, and I need to target IE and Firefox as well.
Is there a method I've overlooked for saving files directly from a web worker? If so, what is it? Or am I just out of luck here?
tl;dr demo
You don't need to copy the entire file to the client side at all. You don't even need to transfer it, in fact. First a recap.
This is how to create Blob from some typed array:
// Some arbitrary binary data
const mydata = new Uint16Array([1,2,3,4,5]);
// mydata vs. mydata.buffer does not seem to make any difference
const blob = new Blob([mydata], {type: "octet/stream"});
You can create an object URL, which is a copy of the original Blob managed by the browser and accessible as URL. I have done this with huge files without seeing performance impact:
const url = URL.createObjectURL(blob);
This is how I typically download URLs:
const link = document.createElement("a");
link.download = "data.bin";
link.href = e.data.link;
link.appendChild(new Text("Download data"));
link.addEventListener("click", function() {
this.parentNode.removeChild(this);
// remember to free the object url, but wait until the download is handled
setTimeout(()=>{URL.revokeObjectURL(e.data.link);}, 500)
});
document.body.appendChild(link);
You can trigger the download automatically by invoking click event on that link. I prefer to let the user decide when to download.
So, all together:
worker.js
// Some arbitrary binary data
const mydata = new Uint16Array([1,2,3,4,5]);
self.onmessage = function(e) {
console.log("Message: ",e.data)
switch(e.data.name) {
case "make-download" :
const blob = new Blob([mydata.buffer], {type: "octet/stream"});
const url = URL.createObjectURL(blob);
self.postMessage({name:"download-link", link:url});
break;
default:
console.error("Unknown message:", e.data.name);
}
}
main.js
var worker = new Worker("worker.js");
worker.addEventListener("message", function(e) {
switch(e.data.name) {
case "download-link" : {
if(e.data.error) {
console.error("Download error: ", e.data.error);
}
else {
const link = document.createElement("a");
link.download = "data.bin";
link.href = e.data.link;
link.appendChild(new Text("Download data"));
link.addEventListener("click", function() {
this.parentNode.removeChild(this);
// remember to free the object url, but wait until the download is handled
setTimeout(()=>{URL.revokeObjectURL(e.data.link);}, 500)
});
document.body.appendChild(link);
}
break;
}
default:
console.error("Unknown message:", e.data.name);
}
});
function requestDownload() {
worker.postMessage({name:"make-download"});
}
When I click Download in my demo, I can see this in my HEX editor:
Looks just fine :)

Resources