I'm pretty new to this type of programming and I'm having some trouble populating an array from a nested call. I'm pretty sure this needs to be done using callbacks, but I'm having trouble wrapping my brain around it. Closures must also come into play here. I tried searching the web for a similar example but didn't find much.
Here is my original code. I tried a few different approaches but didn't pull it off.
TaskSchema.statics.formatAssignee = function(assignees) {
var users = [];
assignees.forEach(function(uid) {
mongoose.model('User').findById(uid, function(err, user) {
users.push({
name: user.name.full
, id: user.id
});
});
});
return users;
}
I really like the following pattern (recursion is the most elegant solution to async loops):
TaskSchema.statics.formatAssignee = function(assignees, callback) {
var acc = []
, uids = assignees.slice()
(function next(){
if (!uids.length) return callback(null, acc);
var uid = uids.pop()
mongoose.model('User').findById(uid, function(err, user) {
if (err) return callback(err);
acc.push({
name: user.name.full
, id: user.id
});
next();
});
})();
}
Check out async, it has an async foreach loop.
Edit
Here is the foreach method from the async library
async.forEach = function (arr, iterator, callback) {
if (!arr.length) {
return callback();
}
var completed = 0;
_forEach(arr, function (x) {
iterator(x, function (err) {
if (err) {
callback(err);
callback = function () {};
}
else {
completed += 1;
if (completed === arr.length) {
callback();
}
}
});
});
};
var _forEach = function (arr, iterator) {
if (arr.forEach) {
return arr.forEach(iterator);
}
for (var i = 0; i < arr.length; i += 1) {
iterator(arr[i], i, arr);
}
};
you could do something like:
Give formatAssignee a callback.
Count down how many users you need to push onto users.
After you push the last one, invoke the callback with the parameter users.
Related
In the code below, when I run in debug mode with a break-point at this line: content.push(data.Body.toString()); I can see that data is inserted to the content array.
However when I run the code normally, content comes back empty.
How can I get it to populate the array for downstream use?
var params = { Bucket: "thebucket", Prefix: "theprefix/" }
var content = [];
function getS3Data()
{
var s3 = new aws.S3();
s3.listObjects(params, function (err, data)
{
if (err) throw err; // an error occurred
else
{
var i;
for (i = 0; i < data.Contents.length; i++)
{
var currentValue = data.Contents[i];
if(currentValue.Key.endsWith(params.Prefix) == false)
{
var goParams = { Bucket: params.Bucket, Key: currentValue.Key };
s3.getObject(goParams, function(err, data)
{
if (err) throw err; //error
content.push(data.Body.toString());
});
};
};
}//else
});//listObjects
}//getS3Data
getS3Data();
console.log(content); //prints empty here when run in non-debug.
The line:
console.log(content)
is being executed before the line:
content.push(data.Body.toString());
the function you are passing as a 2nd argument to s3.listObjects will be executed asynchronously. If you want to log out content you need to do it within the callback function meaning:
s3.listObjects(params, function (err, data) {
if (err) throw err;
else {
// ...
console.log(content)
}
});
A better approach would be to implement getS3Data with Promise so you can run code after the object listing is done for sure.
function getS3Data() {
return new Promise((resolve, reject) => {
if (err) {
reject(err)
} else {
const promises = []
for (const i = 0; i < data.Contents.length; i++) {
const currentValue = data.Contents[i];
if (currentValue.Key.endsWith(params.Prefix) == false) {
const goParams = { Bucket: params.Bucket, Key: currentValue.Key };
promises.push(new Promise((res, rej) => {
s3.getObject(goParams, function (err, data) {
if (err) {
rej(err); //error
} else {
res(data.Body.toString());
}
});
}));
}
}
Promise.all(promises).then(resolve);
}
});
}
getS3Data()
.then(result => { // this will actually be `content` from your code example
console.log(result);
}).catch(error => {
console.error(error);
})
Node.js' documentation has an example very similar to the problem you are experiencing:
Dangers of Mixing Blocking and Non-Blocking Code
The issue arises because the variable content is not set as soon as getS3Data has finished, because it is an asynchronous function. content will be set some time later. But your call to console.log(content); will execute immediately after getS3Data has finished, so at that point content has not been set yet.
You can test that by adding an extra log:
s3.getObject(goParams, function(err, data)
{
if (err) throw err; //error
content.push(data.Body.toString());
console.log("Content has been assigned");
});
And then change the bottom to:
getS3Data();
console.log("getS3Data has finished", content);
It's likely you'll get the messages in this order:
getS3Data has finished
Content has been assigned
I am trying to make a node express app where I fetch data from different url's making a call to node-fetch to pull the body of some pages and other information about certain url endpoints. I want to then render a html table to display this data through an array of information. I am having trouble with the call to render the information as all the functions are asynchronous making it difficult to make sure all the promise calls have been resolved before making my call to render the page. I have been looking into using bluebird and other promise calls of .finally() and .all() but they don't seem to work on my data as it is not an array of promise calls, but an array of objects. Each object was 4 promise calls to fetch data relating to a column of my table all in one row. Is there a function or specific way to render the page after all promises are resolved?
var express = require('express');
var fetch = require('node-fetch');
fetch.Promise = require('bluebird');
var router = express.Router();
const client = require('../platform-support-tools');
function makeArray() {
var registry = client.getDirectory();
var data_arr = [];
for (var i = 0; i < registry.length; i++) {
var firstUp = 0;
for (var j = 0; i < registry[i]; j++) {
if (registry[i][j]['status'] == 'UP') {
firstUp = j;
break;
}
}
var object = registry[i][firstUp];
data_arr.push({
'name': object['app'],
'status': object['status'],
'swagUrl': object['homePageUrl'] + 'swagger-ui.html',
'swag': getSwag(object),
'version': getVersion(object['statusPageUrl']),
'timestamp': getTimestamp(object['statusPageUrl']),
'description': getDescription(object['healthCheckUrl'])
});
}
return data_arr;
}
function getSwag(object_in) {
var homeUrl = object_in['homePageUrl'];
if (homeUrl[homeUrl.length - 1] != '/'){
homeUrl += '/';
}
var datum = fetch(homeUrl + 'swagger-ui.html')
.then(function (res) {
return res.ok;
}).catch(function (err) {
return 'none';
});
return datum;
}
function getVersion(url_in) {
var version = fetch(url_in)
.then(function(res) {
return res.json();
}).then(function(body) {
return body['version'];
}).catch(function (error) {
return 'none';
});
return version;
}
function getTimestamp(url_in) {
var timestamp = fetch(url_in)
.then(function(res) {
return res.json();
}).then(function(body) {
return body['timestamp'];
}).then(function (res) {
return body['version'];
}).catch(function (error) {
return 'none';
});
return timestamp;
}
function getDescription(url_in) {
var des = fetch(url_in)
.then(function(res) {
return res.json();
}).then(function(body) {
return body['description'];
}).catch(function (error) {
return 'none';
});
return des;
}
/* GET home page. */
router.get('/', function (req, res, next) {
var data_arr = makeArray();
Promise.all(data_arr)
.then(function (response) {
//sorting by app name alphabetically
response.sort(function (a, b) {
return (a.name > b.name) ? 1 : ((b.name > a.name) ? -1 : 0);
});
res.render('registry', {title: 'Service Registry', arr: response})
}).catch(function (err) {
console.log('There was an error loading the page: '+err);
});
});
To wait on all those promises, you will have to put them into an array so you can use Promise.all() on them. You can do that like this:
let promises = [];
for (item of data_arr) {
promises.push(item.swag);
promises.push(item.version);
promises.push(item.timestamp);
promises.push(item.description);
}
Promise.all(promises).then(function(results) {
// all promises done here
})
If you want the values from all those promises, back into the object that's a bit more work.
let promises = [];
for (item of data_arr) {
promises.push(item.swag);
promises.push(item.version);
promises.push(item.timestamp);
promises.push(item.description);
}
Promise.all(promises).then(function(results) {
// replace promises with their resolved values
let index = 0;
for (let i = 0; i < results.length; i += 4) {
data_arr[index].swag = results[i];
data_arr[index].version = results[i + 1];
data_arr[index].timestamp = results[i + 2];
data_arr[index].description = results[i + 3];
++index;
});
return data_arr;
}).then(function(data_arr) {
// process results here in the array of objects
});
If you had to do this more often that just this once, you could remove the hard coding of property names and could iterate all the properties, collect the property names that contain promises and automatically process just those.
And, here's a more general version that takes an array of objects where some properties on the objects are promises. This implementation modifies the promise properties on the objects in place (it does not copy the array of the objects).
function promiseAllProps(arrayOfObjects) {
let datum = [];
let promises = [];
arrayOfObjects.forEach(function(obj, index) {
Object.keys(obj).forEach(function(prop) {
let val = obj[prop];
// if it smells like a promise, lets track it
if (val && val.then) {
promises.push(val);
// and keep track of where it came from
datum.push({obj: obj, prop: prop});
}
});
});
return Promise.all(promises).then(function(results) {
// now put all the results back in original arrayOfObjects in place of the promises
// so now instead of promises, the actaul values are there
results.forEach(function(val, index) {
// get the info for this index
let info = datum[index];
// use that info to know which object and which property this value belongs to
info.obj[info.prop] = val;
});
// make resolved value be our original (now modified) array of objects
return arrayOfObjects;
});
}
You would use this like this:
// data_arr is array of objects where some properties are promises
promiseAllProps(data_arr).then(function(r) {
// r is a modified data_arr where all promises in the
// array of objects were replaced with their resolved values
}).catch(function(err) {
// handle error
});
Using the Bluebird promise library, you can make use of both Promise.map() and Promise.props() and the above function would simply be this:
function promiseAllProps(arrayOfObjects) {
return Promise.map(arrayOfObjects, function(obj) {
return Promise.props(obj);
});
}
Promise.props() iterates an object to find all properties that have promises as values and uses Promise.all() to await all those promises and it returns a new object with all the original properties, but the promises replaced by the resolved values. Since we have an array of objects, we use Promise.map() to iterate and await the whole array of those.
Using AngularFire, Angular, Firebase.
I load a list of users from a Firebase Database. I use $loaded to ensure it waits until data loads.
I take this list, compare it against another firebase database of groups and push the results into two arrays.
Based on the console.logs the data sorts correctly. However, inside my template I get a blank page (I think this is because the page loads before the data is sorted).
Thoughts?
let userLoggedIn = AuthFactory.getUser();
var allUsersArray = $firebaseArray(ConnectFactory.fbUserDb);
var x = firebase.database().ref('groups');
var friendArr = [];
var notFriendArr = [];
allUsersArray.$loaded().then(function(){
angular.forEach(allUsersArray, function(user, i) {
var haveIAdded = x.child(userLoggedIn).child(allUsersArray[i].uid).once('value').then(function (snap) {
if (snap.val() !== null) {
return true;
} else {
return false;
}
});
var haveTheyAdded = x.child(allUsersArray[i].uid).child(userLoggedIn).once('value').then(function (snap) {
if (snap.val() !== null) {
return true;
} else {
return false;
}
});
Promise.all([haveIAdded, haveTheyAdded]).then(function([you, they]) {
if (you && they) {
console.log('We Are Friends', allUsersArray[i]);
friendArr.push(allUsersArray[i]);
} else {
console.log('not a friend ', allUsersArray[i]);
notFriendArr.push(allUsersArray[i]);
}
});
});
$scope.friendList = friendArr;
$scope.notFriendList = notFriendArr;
});
Alright, this time I tried to actually read the question before attempting to answer. ;-)
When you set your $scope.friendList and $scope.notFriendList within the $loaded promise, your Promise.all may (and most likely) havn't resolved yet when those are called, since angular.forEach doesn't wait for the promises to finish before moving on to the next statement in the function. So you'll have to build an array of promises and wait for them all to resolve outside of the loop before attempting to set your $scope variables.
allUsersArray.$loaded().then(function(){
var promises = [];
var friendArr = [];
var notFriendArr = [];
angular.forEach(allUsersArray, function(user, i) {
... // Same as before
promises.push(
Promise.all([haveIAdded, haveTheyAdded]).then(function([you, they]) {
if (you && they) {
console.log('We Are Friends', allUsersArray[i]);
friendArr.push(allUsersArray[i]);
} else {
console.log('not a friend ', allUsersArray[i]);
notFriendArr.push(allUsersArray[i]);
}
})
);
});
Promise.all(promises).then(function(){
$scope.friendList = friendArr;
$scope.notFriendList = notFriendArr;
});
});
I want to compare the data which I got from Mongo to javascript array. I am using lodash to compare. But it always return incorrect result.
var editUser = function(userData, getOutFunction) {
var status = CONSTANTS.NG;
checkExistUser(userData._id).then(function(user) {
if (user !== null) {
var userGroup = JSON.stringify(user.group);
user.group = user.group.map((groupId) => {
return groupId.toString();
});
var removedGroups = _.difference(userGroup, userData.group);
var addedGroups = _.difference(userData.group, userGroup);
console.log('Removed Groups: ', removedGroups);
console.log('Added Groups: ', addedGroups);
} else {
status = CONSTANTS.NG;
logger.debug(DEBUG_CLASS_NAME, "Cannot find object");
if (typeof(getOutFunction) !== 'undefined') {
getOutFunction(status, null);
} else {
NO_CALLBACK();
}
}
}).catch(function() {
console.log('Promise is error');
});
var checkExistUser = function(userId) {
return new Promise(function(resolve, reject) {
UserDAO.findById(userId, function(err, user) {
if (err) {
logger.debug(DEBUG_CLASS_NAME, {
name: err.name,
code: err.code,
message: err.message,
method: "checkExist"
});
resolve(null);
} else {
resolve(user);
}
});
});
}
For example:When I try to input value for lodash difference function
var user.group = ["58b8da67d585113517fed34e","58b8da6ed585113517fed34f"];
var userData.group = [ '58b8da67d585113517fed34e' ];
I want lodash difference return below result:
Removed Groups: ['58b8da6ed585113517fed34f']
Added Groups: []
However, the function gave me the result like:
Removed Groups: []
Added Groups: [ '58b8da67d585113517fed34e' ]
Can anyone help me in this case?
I will do appreciate it.
I have had this issue as well, the result from mongodb is an ObjectId type so you can compare the someObjectId.toString() value with your array of strings, or you could use
someObjectId.equals(stringOrObjectIdValue)
However, if you want to keep using lodash functions you will either have to force both arrays to strings or to ObjectIds before passing them into the function.
I'm having some problems with one async process on nodejs.
I'm getting some data from a remote JSON and adding it in my array, this JSON have some duplicated values, and I need check if it already exists on my array before add it to avoid data duplication.
My problem is when I start the loop between the JSON values, the loop call the next value before the latest one be process be finished, so, my array is filled with duplicated data instead of maintain only one item per type.
Look my current code:
BookRegistration.prototype.process_new_books_list = function(data, callback) {
var i = 0,
self = this;
_.each(data, function(book) {
i++;
console.log('\n\n ------------------------------------------------------------ \n\n');
console.log('BOOK: ' + book.volumeInfo.title);
self.process_author(book, function() { console.log('in author'); });
console.log('\n\n ------------------------------------------------------------');
if(i == data.length) callback();
})
}
BookRegistration.prototype.process_author = function(book, callback) {
if(book.volumeInfo.authors) {
var author = { name: book.volumeInfo.authors[0].toLowerCase() };
if(!this.in_array(this.authors, author)) {
this.authors.push(author);
callback();
}
}
}
BookRegistration.prototype.in_array = function(list, obj) {
for(i in list) { if(list[i] === obj) return true; }
return false;
}
The result is:
[{name: author1 }, {name: author2}, {name: author1}]
And I need:
[{name: author1 }, {name: author2}]
UPDATED:
The solution suggested by #Zub works fine with arrays, but not with sequelize and mysql database.
When I try to save my authors list on the database, the data is duplicated, because the system started to save another array element before finish to save the last one.
What is the correct pattern on this case?
My code using database is:
BookRegistration.prototype.process_author = function(book, callback) {
if(book.volumeInfo.authors) {
var author = { name: book.volumeInfo.authors[0].toLowerCase() };
var self = this;
models.Author.count({ where: { name: book.volumeInfo.authors[0].toLowerCase() }}).success(function(count) {
if(count < 1) {
models.Author.create(author).success(function(author) {
console.log('SALVANDO AUTHOR');
self.process_publisher({ book:book, author:author }, callback);
});
} else {
models.Author.find({where: { name: book.volumeInfo.authors[0].toLowerCase() }}).success(function(author) {
console.log('FIND AUTHOR');
self.process_publisher({ book:book, author:author }, callback);
});
}
});
// if(!this.in_array(this.authors, 'name', author)) {
// this.authors.push(author);
// console.log('AQUI NO AUTHOR');
// this.process_publisher(book, callback);
// }
}
}
How can I avoid data duplication in an async process?
This is because you are comparing different objects and result is always false.
Just for experiment type in the console:
var obj1 = {a:1};
var obj2 = {a:1};
obj1 == obj2; //false
When comparing objects (as well as arrays) it only results true when obj1 links to obj2:
var obj1 = {a:1};
var obj2 = obj1;
obj1 == obj2; //true
Since you create new author objects in each process_author call you always get false when comparing.
In your case the solution would be to compare name property for each book:
BookRegistration.prototype.in_array = function(list, obj) {
for(i in list) { if(list[i].name === obj.name) return true; }
return false;
}
EDIT (related to your comment question):
I would rewrite process_new_books_list method as follows:
BookRegistration.prototype.process_new_books_list = function(data, callback) {
var i = 0,
self = this;
(function nextBook() {
var book = data[i];
if (!book) {
callback();
return;
}
self.process_author(book, function() {
i++;
nextBook();
});
})();
}
In this case next process_author is being called not immediately (like with _.each), but after callback is executed, so you have consequence in your program.
Not sure is this works though.
Sorry for my English, I'm not a native English speaker