I have an array in which project _id is stored in the user collection. I'm initially fetching that array and then I'm trying to retrieve the data corresponding to the _id (stored in user collection) stored if projects collection. The data retrieved is pushed on to the array and that array is returned as an output.
But I'm unable to do so. An empty array is being returned.
var projectInfo= new Array();
users.post('/retrieveProjects', function(user, res, next) {
MongoClient.connect(url, function(err,db) {
if (err) throw err;
var dbo = db.db("EMWorks");
dbo.collection("users").findOne({_id:user.body.userid})
.then(response => {
if(response)
{
console.log(response);
(response.project).forEach(element => {
dbo.collection("projects").findOne({_id:element})
.then(respo => {
console.log(respo);
projectInfo.push(respo);
});
console.log(projectInfo) ;
});
}else{
console.log("No Projects created yet...!");
}
});
});
});
Output of the code
The problem is that dbo.collection("projects").findOne is async and returns a promise.
The forEach loop runs and generates these promises for each elemnt, but does not wait for its execution.
The promises are not resolved when the loop finishes and an empty array is returned.
You can wait for the promises to be executed and return the array only after that:
const promises = [];
(response.project).forEach(element => {
promises.push(dbo.collection("projects").findOne({_id:element})
.then(respo => {
console.log(respo);
projectInfo.push(respo);
});
}));
Promise.all(promises).then(function(values) {
console.log(projectInfo) ;
});
This can also be simplified with the map function and async/await :
const promises = (response.project).map(async element => {
const respo = await dbo.collection("projects").findOne({_id:element});
projectInfo.push(respo);
});
Promise.all(promises).then(function(values) {
console.log(projectInfo) ;
});
Related
The problem
At web app startup, the main task is to retrieve all user informations that are stored in various Firestore's collections and sub-collections.
The problem is that, even if I uses promises and a Loading State to prevent an empty app rendering, the app is rendered with all collections data, except for collections that have sub-collections.
The process
- If user is logged
-- Set Loading Status Active
-- Load Collection A
-- Load Collection B and forEach, load all sub-collection
-- Load Collection C
-- Set Loading Status Inactive
At this point, the app is rendered, but only with Collection A and Collection C. The collection B is loaded (I can see it by Redux Logs) but can't be seen in the app.
Those datas appears only if I change the component status (open/close a menu, for example).
Some code
Here's how I retrieve a collection with sub-collections:
export function setCompanyJobs(user) {
return {
type: "SET_COMPANY_JOBS",
payload: loadCompanyJobs(user),
};
}
Function that retrieve the main collection
export function loadCompanyJobs(user) {
return new Promise((resolve, reject) => {
let companyJobs = [];
db.collection("company").doc(user.selectedCompany).collection("jobs").get().then((jobs) => {
jobs.forEach((job) => {
loadJobLinkedServices(user, job).then((jobLinkedServices) => {
companyJobs.push({
id: job.id,
...
});
});
});
resolve(companyJobs);
}).catch(function (error) {
...
});
});
}
Function that retrieve all collection's sub-collections
export function loadJobLinkedServices(user, job){
return new Promise((resolve, reject) => {
let jobLinkedServices = [];
db.collection("company").doc(user.selectedCompany).collection("jobs").doc(job.id).collection("linkedServices").get().then((linkedServices) => {
linkedServices.forEach((linkedService) => {
jobLinkedServices.push({
id: linkedService.id,
...
});
});
resolve(jobLinkedServices)
}).catch(function (error) {
...
});
})
When you do
return new Promise((resolve, reject) => {
let companyJobs = [];
db.collection("company").doc(user.selectedCompany).collection("jobs").get().then((jobs) => {
jobs.forEach((job) => {
loadJobLinkedServices(user, job).then((jobLinkedServices) => {
companyJobs.push({
id: job.id,
...
});
});
});
resolve(companyJobs);
}).catch(function (error) {
...
});
});
nothing ensure that your Promise resolves only after ALL the queries triggered in the jobs.forEach() loop are done (i.e. the promises returned by the calls to the loadJobLinkedServices function have resolved).
I don't know reactjs but I think you can use the JavaScript Promise.all() method along the following lines:
return new Promise((resolve, reject) => {
let promises = [];
let companyJobs = [];
db.collection("company").doc(user.selectedCompany).collection("jobs").get().then((jobs) => {
jobs.forEach((job) => {
promises.push(loadJobLinkedServices(user, job));
});
Promise.all(promises).
then(results => {
//Loop over the results array to populate the companyJobs array
resolve(companyJobs);
})
}).catch(function (error) {
...
});
});
Also, don't forget to correctly chain your calls to the different asynchronous functions, i.e. something like:
query Collection A
THEN query Collection B
THEN query all sub-collections (with Promise.all())
THEN query Collection C
THEN set Loading Status Inactive
Finally, a last remark: Note that the get() method returns a promise, so I am not sure that you need to wrap the calls to the get() method into some new Promises (Again, I am not versed in reactjs, so this remark may be wrong).
In other words, I think you could do something like the following (for example for the loadJobLinkedServices function):
export function loadJobLinkedServices(user, job){
let jobLinkedServices = [];
return db.collection("company").doc(user.selectedCompany).collection("jobs").doc(job.id).collection("linkedServices").get()
.then((linkedServices) => {
linkedServices.forEach((linkedService) => {
jobLinkedServices.push({
id: linkedService.id,
...
});
});
return jobLinkedServices;
}).catch(function (error) {
...
});
})
I am trying to run some API calls in parallel, but am having problems since I am trying to call a function again before the API data has been returned.
I am thinking that I could possibly use the new command in Node, but am not sure how to structure it into this scheme. I am trying to avoid recursion, as I already have a recursive version working and it is slow.
Currently I am trying to this code on the server.
loopThroughArray(req, res) {
for(let i=0; i<req.map.length; i++) {
stack[i] = (callback) => {
let data = getApi(req, res, req.map[i], callback)
}
}
async.parallel(stack, (result) => {
res.json(result)
})
}
....
function getApi(req, res, num, cb) {
request({
url: 'https://example.com/api/' + num
},
(error, response, body) => {
if(error) {
// Log error
} else {
let i = {
name: JSON.parse(body)['name'],
age: '100'
}
console.log(body) // Returns empty value array.length > 1 (req.map[i])
cb(i)
}
})
Is there a way to spawn new instances of the function each time it's called and accumulate the results to send back as one result to the client?
Here's an example of calling Web APIs (each with different parameters), using the Async library, we start by creating an array of N function variables.
const async = require('async');
const request = require('request');
//Set whatever request options you like, see: https://github.com/request/request#requestoptions-callback
var requestArray = [
{url: 'https://httpbin.org/get'},
{url: 'https://httpbin.org/ip'}
];
let getApi = function (opt, callback) {
request(opt, (err, response, body) => {
callback(err, JSON.parse(body));
});
};
const functionArray = requestArray.map((opt) => {
return (callback) => getApi(opt, callback);
});
async.parallel(
functionArray, (err, results) => {
if (err) {
console.error('Error: ', err);
} else {
console.log('Results: ', results.length, results);
}
});
You can easily switch the Url and Query values to match whatever you need. I'm using HttpBin here, since it's good for illustrative purposes.
In the code below, when I run in debug mode with a break-point at this line: content.push(data.Body.toString()); I can see that data is inserted to the content array.
However when I run the code normally, content comes back empty.
How can I get it to populate the array for downstream use?
var params = { Bucket: "thebucket", Prefix: "theprefix/" }
var content = [];
function getS3Data()
{
var s3 = new aws.S3();
s3.listObjects(params, function (err, data)
{
if (err) throw err; // an error occurred
else
{
var i;
for (i = 0; i < data.Contents.length; i++)
{
var currentValue = data.Contents[i];
if(currentValue.Key.endsWith(params.Prefix) == false)
{
var goParams = { Bucket: params.Bucket, Key: currentValue.Key };
s3.getObject(goParams, function(err, data)
{
if (err) throw err; //error
content.push(data.Body.toString());
});
};
};
}//else
});//listObjects
}//getS3Data
getS3Data();
console.log(content); //prints empty here when run in non-debug.
The line:
console.log(content)
is being executed before the line:
content.push(data.Body.toString());
the function you are passing as a 2nd argument to s3.listObjects will be executed asynchronously. If you want to log out content you need to do it within the callback function meaning:
s3.listObjects(params, function (err, data) {
if (err) throw err;
else {
// ...
console.log(content)
}
});
A better approach would be to implement getS3Data with Promise so you can run code after the object listing is done for sure.
function getS3Data() {
return new Promise((resolve, reject) => {
if (err) {
reject(err)
} else {
const promises = []
for (const i = 0; i < data.Contents.length; i++) {
const currentValue = data.Contents[i];
if (currentValue.Key.endsWith(params.Prefix) == false) {
const goParams = { Bucket: params.Bucket, Key: currentValue.Key };
promises.push(new Promise((res, rej) => {
s3.getObject(goParams, function (err, data) {
if (err) {
rej(err); //error
} else {
res(data.Body.toString());
}
});
}));
}
}
Promise.all(promises).then(resolve);
}
});
}
getS3Data()
.then(result => { // this will actually be `content` from your code example
console.log(result);
}).catch(error => {
console.error(error);
})
Node.js' documentation has an example very similar to the problem you are experiencing:
Dangers of Mixing Blocking and Non-Blocking Code
The issue arises because the variable content is not set as soon as getS3Data has finished, because it is an asynchronous function. content will be set some time later. But your call to console.log(content); will execute immediately after getS3Data has finished, so at that point content has not been set yet.
You can test that by adding an extra log:
s3.getObject(goParams, function(err, data)
{
if (err) throw err; //error
content.push(data.Body.toString());
console.log("Content has been assigned");
});
And then change the bottom to:
getS3Data();
console.log("getS3Data has finished", content);
It's likely you'll get the messages in this order:
getS3Data has finished
Content has been assigned
I am creating an array, after it is created that array is resolved as a Object.
However as I need to sort tempArray based on a particular value after the resolve I need it as an array, not an Object.
Also because after the sort I am displaying it using: *ngFor.
Or is it better to somehow sort the Object instead of changing it back to an array?
The console.log of the returned Promise is displayed underneath.
var tempArray = []
return new Promise((resolve, reject) => {
let error = false;
var i = 0;
for (let key of arrayOfPosts) {
tempArray[i] = {
"_id": key._id,
"postPub": key.postPub,
"timePast": this.timePastPublix
}
i++
}
if (error) {
reject('error')
} else {
resolve(tempArray)
}
});
{_id: "5aba1af4c97ce42ea81893bc", postPub: "0.00", timePast: "1970-01-01T01:00:00+01:00"}
{_id: "5ac698172910d705ecd66378", postPendingPayoutPublic: "0.00", timePast: "1970-01-01T01:00:00+01:00"}
You can just return it as an array, no need for an Object:
var promise = new Promise((resolve, reject) => {
resolve([1,2,3,4,5]);
});
// reject example
var promise2 = new Promise((resolve, reject) => {
var array = [];
if(array.length == 0)
reject("error");
else
resolve(array);
});
promise.then(data => console.log(data));
promise2.then(data => console.log(data))
.catch(error => console.log(error));
Here is a stackblitz which uses the promise array return values in an *ngFor: https://stackblitz.com/edit/angular-vpkk1a
I am trying to make a node express app where I fetch data from different url's making a call to node-fetch to pull the body of some pages and other information about certain url endpoints. I want to then render a html table to display this data through an array of information. I am having trouble with the call to render the information as all the functions are asynchronous making it difficult to make sure all the promise calls have been resolved before making my call to render the page. I have been looking into using bluebird and other promise calls of .finally() and .all() but they don't seem to work on my data as it is not an array of promise calls, but an array of objects. Each object was 4 promise calls to fetch data relating to a column of my table all in one row. Is there a function or specific way to render the page after all promises are resolved?
var express = require('express');
var fetch = require('node-fetch');
fetch.Promise = require('bluebird');
var router = express.Router();
const client = require('../platform-support-tools');
function makeArray() {
var registry = client.getDirectory();
var data_arr = [];
for (var i = 0; i < registry.length; i++) {
var firstUp = 0;
for (var j = 0; i < registry[i]; j++) {
if (registry[i][j]['status'] == 'UP') {
firstUp = j;
break;
}
}
var object = registry[i][firstUp];
data_arr.push({
'name': object['app'],
'status': object['status'],
'swagUrl': object['homePageUrl'] + 'swagger-ui.html',
'swag': getSwag(object),
'version': getVersion(object['statusPageUrl']),
'timestamp': getTimestamp(object['statusPageUrl']),
'description': getDescription(object['healthCheckUrl'])
});
}
return data_arr;
}
function getSwag(object_in) {
var homeUrl = object_in['homePageUrl'];
if (homeUrl[homeUrl.length - 1] != '/'){
homeUrl += '/';
}
var datum = fetch(homeUrl + 'swagger-ui.html')
.then(function (res) {
return res.ok;
}).catch(function (err) {
return 'none';
});
return datum;
}
function getVersion(url_in) {
var version = fetch(url_in)
.then(function(res) {
return res.json();
}).then(function(body) {
return body['version'];
}).catch(function (error) {
return 'none';
});
return version;
}
function getTimestamp(url_in) {
var timestamp = fetch(url_in)
.then(function(res) {
return res.json();
}).then(function(body) {
return body['timestamp'];
}).then(function (res) {
return body['version'];
}).catch(function (error) {
return 'none';
});
return timestamp;
}
function getDescription(url_in) {
var des = fetch(url_in)
.then(function(res) {
return res.json();
}).then(function(body) {
return body['description'];
}).catch(function (error) {
return 'none';
});
return des;
}
/* GET home page. */
router.get('/', function (req, res, next) {
var data_arr = makeArray();
Promise.all(data_arr)
.then(function (response) {
//sorting by app name alphabetically
response.sort(function (a, b) {
return (a.name > b.name) ? 1 : ((b.name > a.name) ? -1 : 0);
});
res.render('registry', {title: 'Service Registry', arr: response})
}).catch(function (err) {
console.log('There was an error loading the page: '+err);
});
});
To wait on all those promises, you will have to put them into an array so you can use Promise.all() on them. You can do that like this:
let promises = [];
for (item of data_arr) {
promises.push(item.swag);
promises.push(item.version);
promises.push(item.timestamp);
promises.push(item.description);
}
Promise.all(promises).then(function(results) {
// all promises done here
})
If you want the values from all those promises, back into the object that's a bit more work.
let promises = [];
for (item of data_arr) {
promises.push(item.swag);
promises.push(item.version);
promises.push(item.timestamp);
promises.push(item.description);
}
Promise.all(promises).then(function(results) {
// replace promises with their resolved values
let index = 0;
for (let i = 0; i < results.length; i += 4) {
data_arr[index].swag = results[i];
data_arr[index].version = results[i + 1];
data_arr[index].timestamp = results[i + 2];
data_arr[index].description = results[i + 3];
++index;
});
return data_arr;
}).then(function(data_arr) {
// process results here in the array of objects
});
If you had to do this more often that just this once, you could remove the hard coding of property names and could iterate all the properties, collect the property names that contain promises and automatically process just those.
And, here's a more general version that takes an array of objects where some properties on the objects are promises. This implementation modifies the promise properties on the objects in place (it does not copy the array of the objects).
function promiseAllProps(arrayOfObjects) {
let datum = [];
let promises = [];
arrayOfObjects.forEach(function(obj, index) {
Object.keys(obj).forEach(function(prop) {
let val = obj[prop];
// if it smells like a promise, lets track it
if (val && val.then) {
promises.push(val);
// and keep track of where it came from
datum.push({obj: obj, prop: prop});
}
});
});
return Promise.all(promises).then(function(results) {
// now put all the results back in original arrayOfObjects in place of the promises
// so now instead of promises, the actaul values are there
results.forEach(function(val, index) {
// get the info for this index
let info = datum[index];
// use that info to know which object and which property this value belongs to
info.obj[info.prop] = val;
});
// make resolved value be our original (now modified) array of objects
return arrayOfObjects;
});
}
You would use this like this:
// data_arr is array of objects where some properties are promises
promiseAllProps(data_arr).then(function(r) {
// r is a modified data_arr where all promises in the
// array of objects were replaced with their resolved values
}).catch(function(err) {
// handle error
});
Using the Bluebird promise library, you can make use of both Promise.map() and Promise.props() and the above function would simply be this:
function promiseAllProps(arrayOfObjects) {
return Promise.map(arrayOfObjects, function(obj) {
return Promise.props(obj);
});
}
Promise.props() iterates an object to find all properties that have promises as values and uses Promise.all() to await all those promises and it returns a new object with all the original properties, but the promises replaced by the resolved values. Since we have an array of objects, we use Promise.map() to iterate and await the whole array of those.