I have a Node.js/Express app that queries a MySQL db within the route and displays the result to the user. My problem is how do I run the queries and block until both queries are done before redirecting the user to the page they requested?
In my example I have 2 queries that need to finish before I render the page. I can get the queries to run synchronously if i nest query 2 inside the 'result' callback of query 1. This however will become very convoluted when the number of queries increase.
How do I go about running multiple (in this case 2) database queries synchronously without nesting the subsequent query in the prior query's 'result' callback?
I've looked at the 'Flow control / Async goodies' in the Node modules and tried flow-js but I can't get it to work with the async queries.
Listed below are the 2 queries that I'm attempting to execute from the '/home' route. Can the Node experts explain the 'right' way to do this.
app.get('/home', function (req,res) {
var user_array = [];
var title_array = [];
// first query
var sql = 'select user_name from users';
db.execute(sql)
.addListener('row', function(r) {
user_array.push( { user_name: r.user_name } );
})
.addListener('result', function(r) {
req.session.user_array = user_array;
});
// second query
var sql = 'select title from code_samples';
db.execute(sql)
.addListener('row', function(r) {
title_array.push( { title: r.title } );
})
.addListener('result', function(r) {
req.session.title_array = title_array;
});
// because the queries are async no data is returned to the user
res.render('home.ejs', {layout: false, locals: { user_name: user_array, title: title_array }});
});
The goal with node is not to care what order things happen in. This can complicate some scenarios. There is no shame in nesting callbacks. Once you are used to how it looks, you may find that you actually prefer that style. I do; it is very clear what order callbacks will fire. You can forgo the anonymous functions to make it less verbose if you have to.
If you are willing to restructure your code a bit, you can use the "typical" nested callback method. If you want to avoid callbacks, there are numerous async frameworks that will try and help you do this. One that you might want to check out is async.js (https://github.com/fjakobs/async.js). Example of each:
app.get('/home', function (req,res) {
var lock = 2;
var result = {};
result.user_array = [];
result.title_array = [];
var finishRequest = function(result) {
req.session.title_array = result.title_array;
req.session.user_array = result.user_array;
res.render('home.ejs', {layout: false, locals: { user_name: result.user_array, title: result.title_array }});
};
// first query
var q1 = function(fn) {
var sql = 'select user_name from users';
db.execute(sql)
.addListener('row', function(r) {
result.user_array.push( { user_name: r.user_name } );
})
.addListener('result', function(r) {
return fn && fn(null, result);
});
};
// second query
var q2 = function(fn) {
var sql = 'select title from code_samples';
db.execute(sql)
.addListener('row', function(r) {
result.title_array.push( { title: r.title } );
})
.addListener('result', function(r) {
return fn && fn(null, result);
});
}
//Standard nested callbacks
q1(function (err, result) {
if (err) { return; //do something}
q2(function (err, result) {
if (err) { return; //do something}
finishRequest(result);
});
});
//Using async.js
async.list([
q1,
q2,
]).call().end(function(err, result) {
finishRequest(result);
});
});
For a one-off, I would probably just use a reference counting type approach. Simply keep track of how many queries you want to execute and render the response when they have all finished.
app.get('/home', function (req,res) {
var lock = 2;
var user_array = [];
var title_array = [];
var finishRequest = function() {
res.render('home.ejs', {layout: false, locals: { user_name: user_array, title: title_array }});
}
// first query
var sql = 'select user_name from users';
db.execute(sql)
.addListener('row', function(r) {
user_array.push( { user_name: r.user_name } );
})
.addListener('result', function(r) {
req.session.user_array = user_array;
lock -= 1;
if (lock === 0) {
finishRequest();
}
});
// second query
var sql = 'select title from code_samples';
db.execute(sql)
.addListener('row', function(r) {
title_array.push( { title: r.title } );
})
.addListener('result', function(r) {
req.session.title_array = title_array;
lock -= 1;
if (lock === 0) {
finishRequest();
}
});
});
An even nicer approach would be to simply call finishRequest() in each 'result' callback an check for non-empty arrays before you render the response. Whether that will work in your case depends on your requirements.
Here's a really easy trick to handle multiple callbacks.
var after = function _after(count, f) {
var c = 0, results = [];
return function _callback() {
switch (arguments.length) {
case 0: results.push(null); break;
case 1: results.push(arguments[0]); break;
default: results.push(Array.prototype.slice.call(arguments)); break;
}
if (++c === count) {
f.apply(this, results);
}
};
};
Example
Usage:
var handleDatabase = after(2, function (res1, res2) {
res.render('home.ejs', { locals: { r1: res1, r2: res2 }):
})
db.execute(sql1).on('result', handleDatabase);
db.execute(sql2).on('result', handleDatabase);
So basically you need reference counting. This is the standard approach in these situations. I actually use this small utility function instead of flow control.
If you want a full blown flow control solution I would recommend futuresJS
I find that the async library is the best for things like this. https://github.com/caolan/async#parallel
I can't test this or anything, so forgive me if there are some typos. I refactored your query function to be reusable. So, calling queryRows will return a function that matches the format of the async module's parallel callback functions. After both queries are complete, it will call the last function and pass the result of the two queries as an argument, which you can read to pass to your template.
function queryRows(col, table) {
return function(cb) {
var rows = [];
db.execute('SELECT ' + col + ' FROM ' + table)
.on('row', function(r) {
rows.push(r)
})
.on('result', function() {
cb(rows);
});
}
}
app.get('/home', function(req, res) {
async.parallel({
users: queryRow('user_name', 'users'),
titles: queryRow('title', 'code_samples')
},
function(result) {
res.render('home.ejs', {
layout: false,
locals: {user_name: result.users, title: result.titles}
});
});
});
There are some solutions here, but in my opinion the best solution is to make the code synchronously in a very easy way.
You could use the "synchonize" package.
Just
npm install synchronize
Then var sync = require(synchronize);
Put logic which should be synchronous into a fiber by using
sync.fiber(function() {
//put your logic here
}
An example for two mysql queries:
var express = require('express');
var bodyParser = require('body-parser');
var mysql = require('mysql');
var sync = require('synchronize');
var db = mysql.createConnection({
host : 'localhost',
user : 'user',
password : 'password',
database : 'database'
});
db.connect(function(err) {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
});
function saveSomething() {
var post = {id: newId};
//no callback here; the result is in "query"
var query = sync.await(db.query('INSERT INTO mainTable SET ?', post, sync.defer()));
var newId = query.insertId;
post = {foreignKey: newId};
//this query can be async, because it doesn't matter in this case
db.query('INSERT INTO subTable SET ?', post, function(err, result) {
if (err) throw err;
});
}
When "saveSomething()" is called, it inserts a row in a main table and receives the last inserted id. After that the code below will be executed. No need for nesting promises or stuff like that.
option one: if all your queries related to each other, create stored procedure, put all your data logic into it and have a single db.execute
option two: if your db uses one connection then commands a guaranteed to be executed serially and you can use this as async helper
db.execute(sql1).on('row', function(r) {
req.session.user_array.push(r.user);
});
db.execute(sql2)
.on('row', function(r) {
req.session.title_array.push(r.title);
})
.on('end'), function() {
// render data from req.session
});
You can use fibers to write pseudo-synchronous code with Node.JS take a look at these tests for DB https://github.com/alexeypetrushin/mongo-lite/blob/master/test/collection.coffee
they are asynchronous but looks like synchronous, more details http://alexeypetrushin.github.com/synchronize
Related
I am trying to run some API calls in parallel, but am having problems since I am trying to call a function again before the API data has been returned.
I am thinking that I could possibly use the new command in Node, but am not sure how to structure it into this scheme. I am trying to avoid recursion, as I already have a recursive version working and it is slow.
Currently I am trying to this code on the server.
loopThroughArray(req, res) {
for(let i=0; i<req.map.length; i++) {
stack[i] = (callback) => {
let data = getApi(req, res, req.map[i], callback)
}
}
async.parallel(stack, (result) => {
res.json(result)
})
}
....
function getApi(req, res, num, cb) {
request({
url: 'https://example.com/api/' + num
},
(error, response, body) => {
if(error) {
// Log error
} else {
let i = {
name: JSON.parse(body)['name'],
age: '100'
}
console.log(body) // Returns empty value array.length > 1 (req.map[i])
cb(i)
}
})
Is there a way to spawn new instances of the function each time it's called and accumulate the results to send back as one result to the client?
Here's an example of calling Web APIs (each with different parameters), using the Async library, we start by creating an array of N function variables.
const async = require('async');
const request = require('request');
//Set whatever request options you like, see: https://github.com/request/request#requestoptions-callback
var requestArray = [
{url: 'https://httpbin.org/get'},
{url: 'https://httpbin.org/ip'}
];
let getApi = function (opt, callback) {
request(opt, (err, response, body) => {
callback(err, JSON.parse(body));
});
};
const functionArray = requestArray.map((opt) => {
return (callback) => getApi(opt, callback);
});
async.parallel(
functionArray, (err, results) => {
if (err) {
console.error('Error: ', err);
} else {
console.log('Results: ', results.length, results);
}
});
You can easily switch the Url and Query values to match whatever you need. I'm using HttpBin here, since it's good for illustrative purposes.
2nd UPDATE
We are implementing a BulkEdit functionality which sends async CRUD requests to a Backend.
So what I require here is a dynamically created set of nested promises.
In an abstract version the data array could look like:
var objArr = [
{
name: 'A',
subs: [
{
id: 1,
_action: 'create'
},
{
id: 2,
_action: 'create'
},
{
id: 3,
_action: 'delete'
}
]
},
{
name: 'B',
subs: [
{
id: 4,
_action: 'create'
},
{
id: 5,
_action: 'put'
}
]
},
{
name: 'C',
subs: []
}
];
I try to illustrate how the requests should be sent for this data following the order given by '_action'.
Get some transaction ID (see below)
As soon as transaction ID is there start to send requests for every Object in the Array given the following rules:
Per Object send all 'delete' requests at once if there are any.
After that or if there weren't any 'delete' requests send all 'put'
requests if there are any.
After 'delete' and/or 'put' send all 'create' requests if there are any.
As soon as all requests for an Object are done, do something per Object.
As soon as all Objects are done, close the Transaction.
How is it possible to create this dynamic nested/non-nested promise chain?
UPDATED CODE contains now Promise Creation
When calling the function below, first a TransactionService gets a transaction id which is required to be sent with each request. When everything is successful, the Transaction will be closed.
My current issue is that promises are not resolved in the correct order (while the OPTIONS preflight requests seem to be) and that this example creates Promises even if they are not required (e.g. for Object 'C' in the example above).
function startIt(objArr) {
TransactionService.getTransaction().then(function (transaction) {
var promiseArray = MyService.submit(objArr, transaction.id);
$q.all(promiseArray).then(function () {
Transactions.closeTransaction(transaction.id, function () {}).then(function () {
});
};
});
}
This is the function for 'submit':
function submit(objArr, transactionId) {
var promises = objArr.map(function (obj) {
return submitWithTransId(transactionId, obj)
.then(function (response) {
// Object done
});
});
return promises;
}
And this function is actually creating the Promises:
function submitWithTransId(transactionId, obj) {
var promisesDelete = [];
var promisesUpdate = [];
var promisesCreate = [];
angular.forEach(obj['subs'], function (sub) {
switch (sub._action) {
case 'delete':
promisesDelete.push(createPromise(sub, bulktransactionId));
break;
case 'put':
promisesUpdate.push(createPromise(sub, bulktransactionId));
break;
case 'create':
promisesCreate.push(createPromise(sub, bulktransactionId));
break;
}
});
var chainedPromises = $q.all(promisesDelete).then(function (deleteResponse) {
return $q.all(promisesUpdate).then(function (updateResponse) {
return $q.all(promisesCreate).then(function (createResponse) {
});
});
});
return chainedPromises;
And this is my createPromise function:
/** only simplified handling create case **/
function createPromise(sub, bulktransactionId) {
var queryParams = {};
if (bulktransactionId !== undefined && bulktransactionId !== null) {
queryParams.transaction_id = bulktransactionId;
}
var promise = MyResourceService.create(queryParams, sub).$promise;
promise.then(function (newSub) {
// do something with the new/updated/deleted sub, especially update view model
});
return promise;
}
/** MyResourceService **/
return $resource(ENV.apiEndpoint + 'api/v1/subs/:_id',
{
_id: '#id'
}, {
create: {
method: 'POST'
}
}
);
You can have a look at the following solution. The objective is to provide you some sort of structure. Please see, you will have to modify to your use.
var objArr = [{
name: 'A',
subs: [{
id: 1,
_action: 'create'
},
{
id: 2,
_action: 'create'
},
{
id: 3,
_action: 'delete'
}
]
},
{
name: 'B',
subs: [{
id: 4,
_action: 'create'
},
{
id: 5,
_action: 'put'
}
]
},
{
name: 'C',
subs: []
}
];
var promises = objArr.map(function(obj) {
return firstLevelPromise(obj)
.then(function(response) {
console.log(response); // promise for each object
return response;
});
});
$q.all(promises)
.then(function(response) {
console.log(response); // completion - close transaction
});
function firstLevelPromise(obj) {
var deletePromises = [];
var putPromies = [];
var insertPromies = [];
obj.subs.forEach(function(sub) { // Preparing promises array for delete, put and insert
if (sub._action === "delete") {
deletePromises.push(deletePromise(sub));
} else if (sub._action === "put") {
putPromies.push(putPromise(sub));
} else {
insertPromies.push(insertPromise(sub));
}
});
return $q.all(deletePromises) // executing delete promises
.then(function(deleteResponse) {
console.log("deleteExecuted: " + obj.name);
return $q.all(putPromies) // on completion of delete, execute put promies
.then(function(putResponse) {
console.log("putExecuted: " + obj.name);
return $q.all(insertPromies) // on completion of put, execute insert promises
.then(function(insertResponse) {
console.log("insertExecuted: " + obj.name);
return "object promise completed: " + obj.name; // on completion, return
});
});
});
}
function deletePromise(task) {
return $q.resolve(task); // write your delete code here
}
function putPromise(task) {
return $q.resolve(task); // write your put code here
}
function insertPromise(task) {
return $q.resolve(task); // write your insert code here
}
Please note, the above code will do the following
Prepare a collection of promises where there is one promise for each object in objectArray
Each promise will have promises chain i.e. delete promises, followed by put promises and finally followed by insert promises i.e. it ensures that for each object perform the delete tasks, then on completion perform the put tasks and then on its completion perform the insert tasks.
Here is a plunker and documentation for $q
UPDATE
The problem is with the createPromise function only. Update your code to following. The problem was that you are calling the then before returning, hence, it is likely that the you are returning a resolved promise like $q.resolve(). In this case, it is possible that a create request gets resolved before delete or put and a put calls gets resolved before delete. Hence, you should return the promise from here and perform the post action things in the $q.all block.
function createPromise(sub, bulktransactionId) {
var queryParams = {};
if (bulktransactionId !== undefined && bulktransactionId !== null) {
queryParams.transaction_id = bulktransactionId;
}
return MyResourceService.create(queryParams, sub).$promise;
}
Try this - the trick is to accumulate the promise (that's what I'm using the reduce for. Hope it helps.
// reversed the order => delete actions first; otherwise you may have to do extra logic may be needed
var objArr = [
{ name: 'A',
subs: [
{ id: null,
_action: 'delete'
},
{ id: 2,
_action: 'create']
}
},
{ name: 'B',
subs: [
{ id: 3.
_action: 'create'
}
]
];
Promise.all(objArr.map(obj => {
return obj.subs.reduce((accumulatedPromisedSub, sub) => {
return accumulatedPromisedSub.then(_ => yourRequestCallHere(sub) )
},
Promise.resolve(true) // you could also do your delete here if you like
)
}))
// OR going sort order agnostic:
Promise.all(objArr.map(obj => {
const deleteAction = obj.subs.find(sub => sub._action === 'delete');
return obj.subs.reduce((accumulatedPromisedSub, sub) => {
if (sub._action === 'delete') return accumulatedPromisedSub;
return accumulatedPromisedSub.then(_ => yourRequestCallHere(sub) )
},
deleteAction ? yourRequestCall(deleteAction) : Promise.resolve(true)
)
}))
I am so happy and thankful, I found it.
To my understanding I had two main issues in my code.
As $resource is not providing a (please forgive me, pros!) real $promise even when I use something like .get().$promise I had to
use bind to map the createPromise function to my arrays
do the mapping only directly before returning the $q.all promise.
In any other case $resource seemed to immediately fill its promise with an empty object and $q.all did not work anymore as promises looked like they where resolved.
Special thanks to #nikhil who supported me in finding this ugly complex thing and who earned the bounty.
This is the working snippet:
function submitWithTransId(transactionId, obj) {
var arrayDelete = [];
var arrayUpdate = [];
var arrayCreate = [];
angular.forEach(obj['subs'], function (sub) {
switch (sub._action) {
case 'delete':
arrayDelete.push(sub);
break;
case 'update':
arrayUpdate.push(sub);
break;
case 'create':
arrayCreate.push(sub);
break;
}
});
var promisesDelete = flattenArray(arrayDelete.map(createPromise.bind(undefined, obj, transactionId)));
return $q.all(promisesDelete).then(function (deleteResponse) {
console.log('Delete Promises ' + obj.name + ' resolved');
var promisesUpdate = flattenArray(arrayUpdate.map(createPromise.bind(undefined, obj, transactionId)));
return $q.all(promisesUpdate).then(function (updateResponse) {
var promisesCreate = flattenArray(arrayCreate.map(createPromise.bind(undefined, obj, transactionId)));
console.log('Update Promises ' + obj.name + ' resolved');
return $q.all(promisesCreate).then(function (createResponse) {
console.log('Create Promises ' + obj.name + ' resolved');
});
});
}).catch(function (error) {
console.log('Catched an error: ');
console.log(error);
});
});
I am using bramski/angular-indexedDB in my application. Basic CRUD operations are working fine, but the custom queries are not working as expected.
I am using the code
angular.module('myModuleName', ['indexedDB'])
.config(function ($indexedDBProvider) {
$indexedDBProvider
.connection('myIndexedDB')
.upgradeDatabase(1, function(event, db, tx){
var objStore = db.createObjectStore('people', {keyPath: 'ssn'});
objStore.createIndex('name_idx', 'age', {unique: false});
objStore.createIndex('name_idx, age_idx', ['name', 'age'] , {unique: false});
});
Basic query operations are working like follows
$indexedDB.openStore('people', function(x){
var find = x.query();
find = find.$eq('John');
find = find.$index("name_idx");
x.eachWhere(find).then(function(e){
$scope.list= e;
});
});
which results following query.
select * from people where name='John'
But, in the above scenario how we can execute custom quires like
select * from people where name='John' and age='25';
delete from people where name='John' and age='25';
The library you are using doesn't have complex queries, however you can write a pure-js solution for it, similar to this:
First you need to define your index as:
objStore.createIndex('name_age_idx', ['name', 'age'] , {unique: false});
Then you can have a search query for only those values that match the search result
searchIndexedDB = function (name, age, callback) {
var request = indexedDB.open(dbName);
request.onsuccess = function(e) {
var db = e.target.result;
var trans = db.transaction(objectStoreName, 'readonly');
var store = trans.objectStore(objectStoreName);
var index = store.index('name_age_idx');
var keyRange = IDBKeyRange.only([name, age]);
// open the index for all objects with the same name and age
var openCursorRequest = index.openCursor(keyRange);
openCursorRequest.onsuccess = function(e) {
let result = e.target.result;
// first check if value is found
if(result){
callback(result.value); // your callback will be called per object
// result.delete() - to delete your object
result.continue(); // to continue itterating - calls the next cursor request
}
};
trans.oncomplete = function(e) {
db.close();
};
openCursorRequest.onerror = function(e) {
console.log("Error Getting: ", e);
};
};
request.onerror = myStorage.indexedDB.onerror;
}
If you need a range from and too index, all you need is change the keyrange to:
var keyRange = IDBKeyRange.bound([name,fromAge], [value, toAge]);
hai I am new to tedious and Es-6,It may be a silly question but I am struggling,
I want the total data in a array, using tedious connections here is my code:
getZipData() {
var Connection = require('tedious').Connection;
Request = require('tedious').Request;
var config = {
userName: 'xx',
password: 'xxxx',
server: 'xxx', // You can use 'localhost\\instance' to connect to named instance
options: {
database: 'xxxxx',
rowCollectionOnDone:'true'
}
}
var connection = new Connection(config);
var jsonArray = [];
connection.on('connect', function (err) {
if (err) {
console.log(err)
}
var sql = "SELECT * FROM xxxxx";
return new Promise(function(resolve,reject){
var request = new Request(sql,
(err, rowCount, rows)=>{
if (err) {
reject(err);
}
else {
alert("rows");
console.log(rowCount + 'rows');
}
});
request.on('row', (columns)=>{
var rowObject = {};
columns.forEach((column)=> {
rowObject[column.metadata.colName] = column.value;
});
jsonArray.push(rowObject);
});
connection.execSql(request);
request.on('done', function(rowCount, more) {
console.log(rowCount + ' rows returned');
alert("jsonArray2:"+jsonArray);
resolve(jsonArray)
});
});
})
}
componentWillMount() {
this.getZipData().then(function(resolved){
console.log(resolved);
alert("data:"+resolved);
}).catch(function(rejected){
console.log(rejected);
})
}
when i add the request.on('done', function(rowCount, more) also i didn't get any data can any one give the solution for it,
I want the total data to be displayed
It looks like you're calling resolve before your query has been executed:
var jsonArray = [];
// Register callback for row event
request.on('row', (columns)=>{
var rowObject = {};
columns.forEach((column)=> {
rowObject[column.metadata.colName] = column.value;
});
jsonArray.push(rowObject);
});
// Call resolve before executing request
resolve(jsonArray);
connection.execSql(request);
The docs mention a done event that indicates a request has completed:
request.on('done', function (rowCount, more, rows) {
// Call resolve here instead?
resolve(jsonArray);
});
Disclaimer: I've haven't actually used Tedious, but from the docs linked this looks like what you're looking for.
I'm pretty new to this type of programming and I'm having some trouble populating an array from a nested call. I'm pretty sure this needs to be done using callbacks, but I'm having trouble wrapping my brain around it. Closures must also come into play here. I tried searching the web for a similar example but didn't find much.
Here is my original code. I tried a few different approaches but didn't pull it off.
TaskSchema.statics.formatAssignee = function(assignees) {
var users = [];
assignees.forEach(function(uid) {
mongoose.model('User').findById(uid, function(err, user) {
users.push({
name: user.name.full
, id: user.id
});
});
});
return users;
}
I really like the following pattern (recursion is the most elegant solution to async loops):
TaskSchema.statics.formatAssignee = function(assignees, callback) {
var acc = []
, uids = assignees.slice()
(function next(){
if (!uids.length) return callback(null, acc);
var uid = uids.pop()
mongoose.model('User').findById(uid, function(err, user) {
if (err) return callback(err);
acc.push({
name: user.name.full
, id: user.id
});
next();
});
})();
}
Check out async, it has an async foreach loop.
Edit
Here is the foreach method from the async library
async.forEach = function (arr, iterator, callback) {
if (!arr.length) {
return callback();
}
var completed = 0;
_forEach(arr, function (x) {
iterator(x, function (err) {
if (err) {
callback(err);
callback = function () {};
}
else {
completed += 1;
if (completed === arr.length) {
callback();
}
}
});
});
};
var _forEach = function (arr, iterator) {
if (arr.forEach) {
return arr.forEach(iterator);
}
for (var i = 0; i < arr.length; i += 1) {
iterator(arr[i], i, arr);
}
};
you could do something like:
Give formatAssignee a callback.
Count down how many users you need to push onto users.
After you push the last one, invoke the callback with the parameter users.