I am trying to run some API calls in parallel, but am having problems since I am trying to call a function again before the API data has been returned.
I am thinking that I could possibly use the new command in Node, but am not sure how to structure it into this scheme. I am trying to avoid recursion, as I already have a recursive version working and it is slow.
Currently I am trying to this code on the server.
loopThroughArray(req, res) {
for(let i=0; i<req.map.length; i++) {
stack[i] = (callback) => {
let data = getApi(req, res, req.map[i], callback)
}
}
async.parallel(stack, (result) => {
res.json(result)
})
}
....
function getApi(req, res, num, cb) {
request({
url: 'https://example.com/api/' + num
},
(error, response, body) => {
if(error) {
// Log error
} else {
let i = {
name: JSON.parse(body)['name'],
age: '100'
}
console.log(body) // Returns empty value array.length > 1 (req.map[i])
cb(i)
}
})
Is there a way to spawn new instances of the function each time it's called and accumulate the results to send back as one result to the client?
Here's an example of calling Web APIs (each with different parameters), using the Async library, we start by creating an array of N function variables.
const async = require('async');
const request = require('request');
//Set whatever request options you like, see: https://github.com/request/request#requestoptions-callback
var requestArray = [
{url: 'https://httpbin.org/get'},
{url: 'https://httpbin.org/ip'}
];
let getApi = function (opt, callback) {
request(opt, (err, response, body) => {
callback(err, JSON.parse(body));
});
};
const functionArray = requestArray.map((opt) => {
return (callback) => getApi(opt, callback);
});
async.parallel(
functionArray, (err, results) => {
if (err) {
console.error('Error: ', err);
} else {
console.log('Results: ', results.length, results);
}
});
You can easily switch the Url and Query values to match whatever you need. I'm using HttpBin here, since it's good for illustrative purposes.
Related
I have an array in which project _id is stored in the user collection. I'm initially fetching that array and then I'm trying to retrieve the data corresponding to the _id (stored in user collection) stored if projects collection. The data retrieved is pushed on to the array and that array is returned as an output.
But I'm unable to do so. An empty array is being returned.
var projectInfo= new Array();
users.post('/retrieveProjects', function(user, res, next) {
MongoClient.connect(url, function(err,db) {
if (err) throw err;
var dbo = db.db("EMWorks");
dbo.collection("users").findOne({_id:user.body.userid})
.then(response => {
if(response)
{
console.log(response);
(response.project).forEach(element => {
dbo.collection("projects").findOne({_id:element})
.then(respo => {
console.log(respo);
projectInfo.push(respo);
});
console.log(projectInfo) ;
});
}else{
console.log("No Projects created yet...!");
}
});
});
});
Output of the code
The problem is that dbo.collection("projects").findOne is async and returns a promise.
The forEach loop runs and generates these promises for each elemnt, but does not wait for its execution.
The promises are not resolved when the loop finishes and an empty array is returned.
You can wait for the promises to be executed and return the array only after that:
const promises = [];
(response.project).forEach(element => {
promises.push(dbo.collection("projects").findOne({_id:element})
.then(respo => {
console.log(respo);
projectInfo.push(respo);
});
}));
Promise.all(promises).then(function(values) {
console.log(projectInfo) ;
});
This can also be simplified with the map function and async/await :
const promises = (response.project).map(async element => {
const respo = await dbo.collection("projects").findOne({_id:element});
projectInfo.push(respo);
});
Promise.all(promises).then(function(values) {
console.log(projectInfo) ;
});
I know how to read from the google finance api, it is pretty simple.
But when I try to write I get the following error:
Error: Request had insufficient authentication scopes
This is my code:
const fs = require('fs');
const readline = require('readline');
const {google} = require('googleapis');
// If modifying these scopes, delete token.json.
const TOKEN_PATH = 'token.json';
// Load client secrets from a local file.
fs.readFile('./GoogleFinanceApi/credentials.json', (err, content) => {
if (err) return console.log('Error loading client secret file:', err);
// Authorize a client with credentials, then call the Google Sheets API.
authorize(JSON.parse(content), appendData);
});
Here ^ in the append data is where I am calling the function, it works when i do the listMajors but not when I do the appendData...
function authorize(credentials, callback) {
const {client_secret, client_id, redirect_uris} = credentials.installed;
const oAuth2Client = new google.auth.OAuth2(
client_id, client_secret, redirect_uris[0]);
// Check if we have previously stored a token.
fs.readFile(TOKEN_PATH, (err, token) => {
if (err) return getNewToken(oAuth2Client, callback);
oAuth2Client.setCredentials(JSON.parse(token));
callback(oAuth2Client);
});
}
function listMajors(auth) {
const sheets = google.sheets({version: 'v4', auth});
sheets.spreadsheets.values.get({
spreadsheetId: '1ckHZsL2fnWVATmXljlewm-6qBo62B0qmu0w_2QdSpGA',
range: 'Sheet1!A2:E',
}, (err, res) => {
if (err) return console.log('The API returned an error: ' + err);
const rows = res.data.values;
if (rows.length) {
console.log('Name, Major:');
// Print columns A and E, which correspond to indices 0 and 4.
rows.map((row) => {
console.log(`${row[0]}, ${row[4]}`);
});
} else {
console.log('No data found.');
}
});
}
function appendData(auth) {
var sheets = google.sheets('v4');
sheets.spreadsheets.values.append({
auth: auth,
spreadsheetId: '1ckHZsL2fnWVATmXljlewm-6qBo62B0qmu0w_2QdSpGA',
range: 'Sheet1!A2:B', //Change Sheet1 if your worksheet's name is something else
valueInputOption: "USER_ENTERED",
resource: {
values: [ ["Void", "Canvas", "Website"], ["Paul", "Shan", "Human"] ]
}
}, (err, response) => {
if (err) {
console.log('The API returned an error: ' + err);
return;
} else {
console.log("Appended");
}
});
}
What am I doing wrong? I have read some posts and they say they didn't add the resource so I tried to fix that but still nothing works...
Probably the issue is in google.sheets in appendData. Perhaps you need to pass auth to google.sheets before you access sheets as how you are doing in listMajors but you are passing auth to the sheets instead of google.sheets. This might be an issue
Can you try below updated code
function appendData(auth) {
const sheets = google.sheets({version: 'v4', auth})
sheets.spreadsheets.values.append({
spreadsheetId: '1ckHZsL2fnWVATmXljlewm-6qBo62B0qmu0w_2QdSpGA',
range: 'Sheet1!A2:B', //Change Sheet1 if your worksheet's name is something else
valueInputOption: "USER_ENTERED",
resource: {
values: [ ["Void", "Canvas", "Website"], ["Paul", "Shan", "Human"] ]
}
}, (err, response) => {
if (err) {
console.log('The API returned an error: ' + err);
return;
} else {
console.log("Appended");
}
});
}
In the code below, when I run in debug mode with a break-point at this line: content.push(data.Body.toString()); I can see that data is inserted to the content array.
However when I run the code normally, content comes back empty.
How can I get it to populate the array for downstream use?
var params = { Bucket: "thebucket", Prefix: "theprefix/" }
var content = [];
function getS3Data()
{
var s3 = new aws.S3();
s3.listObjects(params, function (err, data)
{
if (err) throw err; // an error occurred
else
{
var i;
for (i = 0; i < data.Contents.length; i++)
{
var currentValue = data.Contents[i];
if(currentValue.Key.endsWith(params.Prefix) == false)
{
var goParams = { Bucket: params.Bucket, Key: currentValue.Key };
s3.getObject(goParams, function(err, data)
{
if (err) throw err; //error
content.push(data.Body.toString());
});
};
};
}//else
});//listObjects
}//getS3Data
getS3Data();
console.log(content); //prints empty here when run in non-debug.
The line:
console.log(content)
is being executed before the line:
content.push(data.Body.toString());
the function you are passing as a 2nd argument to s3.listObjects will be executed asynchronously. If you want to log out content you need to do it within the callback function meaning:
s3.listObjects(params, function (err, data) {
if (err) throw err;
else {
// ...
console.log(content)
}
});
A better approach would be to implement getS3Data with Promise so you can run code after the object listing is done for sure.
function getS3Data() {
return new Promise((resolve, reject) => {
if (err) {
reject(err)
} else {
const promises = []
for (const i = 0; i < data.Contents.length; i++) {
const currentValue = data.Contents[i];
if (currentValue.Key.endsWith(params.Prefix) == false) {
const goParams = { Bucket: params.Bucket, Key: currentValue.Key };
promises.push(new Promise((res, rej) => {
s3.getObject(goParams, function (err, data) {
if (err) {
rej(err); //error
} else {
res(data.Body.toString());
}
});
}));
}
}
Promise.all(promises).then(resolve);
}
});
}
getS3Data()
.then(result => { // this will actually be `content` from your code example
console.log(result);
}).catch(error => {
console.error(error);
})
Node.js' documentation has an example very similar to the problem you are experiencing:
Dangers of Mixing Blocking and Non-Blocking Code
The issue arises because the variable content is not set as soon as getS3Data has finished, because it is an asynchronous function. content will be set some time later. But your call to console.log(content); will execute immediately after getS3Data has finished, so at that point content has not been set yet.
You can test that by adding an extra log:
s3.getObject(goParams, function(err, data)
{
if (err) throw err; //error
content.push(data.Body.toString());
console.log("Content has been assigned");
});
And then change the bottom to:
getS3Data();
console.log("getS3Data has finished", content);
It's likely you'll get the messages in this order:
getS3Data has finished
Content has been assigned
I'm trying to upload a csv file using ng-file-upoad. Here is my code snippet:
Upload.upload({
url: baseUrl + '/file-upload',
data: {
file: file
}
})
.then(function(res) {
console.log('success: ===> ', res);
}, function(err) {
console.log('erroir: ===> ', err);
}, function() {
console.log('progress: ', arguments);
});
And in node environment I'm parsing the file and inserting the data in database. I don't want to close the connection. That's why I used "response.write". Here is my code snippet:
var path = req.files.file.path,
currentIndex = 0;
fs.readFile(path, 'utf8', function(err, data) {
if(err) {
// handle error
} else {
// making array (dataArray) from data
dataArray.forEach(function(eachData){
newEntry = new app.db.models.SomeCollection(eachData);
newEntry.save(function(err, data) {
if (currentIndex === dataArray.length) {
res.end('DONE!');
} else {
currentIndex++;
res.write(JSON.stringify({
total: dataArray.length,
done: currentIndex
}));
}
});
})
}
});
My question is how I will get the data I'm passing in "res.write"? I don't want to use socket for only this purpose. Am I missing something?
As already explained here:
response.send(msg) is equal to response.write(msg);response.end();
Which means, send can only be called once, write can be called many times, but you must call end yourself.
You are probably not receiving the response because response.end() is missing.
Once you end() your response you should be able to access the response data in your angular controller in the Upload.upload promise that is returned.
It's not like close a connection as you said. This is not a socket-ish like implementation (such as ws or socket.io). Once a request is made it should have a response even if it is to provide error details about that request (i.e. status 401, 403, 404, etc).
in your angular component:
...
constructor(private incrementalService: IncrementalService) {}
incrementalTest() { //activate with a button or whatnot
this.incrementalService.increment().subscribe( (result:any) => {
if (result.partialText) {
console.log(partialText); //do whatever you need to do with your partial results here!
}
})
}
your angular service:
import { HttpClient, HttpHeaders } from '#angular/common/http';
public class IncrementalService {
constructor(private http: HttpClient) {}
increment(): Observable<ArrayBuffer> {
const options = {
reportProgress: true,
responseType: 'text',
observe: 'events'
}
return this.http.request('get', 'http://someURL', { ...this.addRawHeaderOptions(), ...options});
}
private addRawHeaderOptions() {
const authHeaders = new HttpHeaders({
'Content-Type': 'application/json',
//authorization, Cache-Control: 'no-cache, Pragma:'no-cache', et al. }
return { headers: authHeaders }
}
}
Finally, your back-end service (this is express, but should work similarly for raw node):
async function(request, response) {
const increments = [ 1,2,3,4 ];
response.set('Content-Type', 'text/html');
for (const value of increments) { //contains async call - not switch-outable for a forEach.
response.write(`increment - ${value} `);
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
await delay(1000)
}
response.status(200).end()
}
browser console output when run:
increment - 1
increment - 1 increment - 2
increment - 1 increment - 2 increment - 3
increment - 1 increment - 2 increment - 3 increment - 4
!!Sorry for any typos - i had to transcribe this from a locked-down machine.
I'm pretty new to this type of programming and I'm having some trouble populating an array from a nested call. I'm pretty sure this needs to be done using callbacks, but I'm having trouble wrapping my brain around it. Closures must also come into play here. I tried searching the web for a similar example but didn't find much.
Here is my original code. I tried a few different approaches but didn't pull it off.
TaskSchema.statics.formatAssignee = function(assignees) {
var users = [];
assignees.forEach(function(uid) {
mongoose.model('User').findById(uid, function(err, user) {
users.push({
name: user.name.full
, id: user.id
});
});
});
return users;
}
I really like the following pattern (recursion is the most elegant solution to async loops):
TaskSchema.statics.formatAssignee = function(assignees, callback) {
var acc = []
, uids = assignees.slice()
(function next(){
if (!uids.length) return callback(null, acc);
var uid = uids.pop()
mongoose.model('User').findById(uid, function(err, user) {
if (err) return callback(err);
acc.push({
name: user.name.full
, id: user.id
});
next();
});
})();
}
Check out async, it has an async foreach loop.
Edit
Here is the foreach method from the async library
async.forEach = function (arr, iterator, callback) {
if (!arr.length) {
return callback();
}
var completed = 0;
_forEach(arr, function (x) {
iterator(x, function (err) {
if (err) {
callback(err);
callback = function () {};
}
else {
completed += 1;
if (completed === arr.length) {
callback();
}
}
});
});
};
var _forEach = function (arr, iterator) {
if (arr.forEach) {
return arr.forEach(iterator);
}
for (var i = 0; i < arr.length; i += 1) {
iterator(arr[i], i, arr);
}
};
you could do something like:
Give formatAssignee a callback.
Count down how many users you need to push onto users.
After you push the last one, invoke the callback with the parameter users.