async inside a for loop nodejs - arrays

Question

I am trying to access an api and I will have to run the api calls several times based on the page numbers I need to iterate, the following is the code which I am using and how can I get the all the response pushed into an array.
as nodeJs is single threaded It is not waiting for the responses from the api.
How can I can tackle this and ensure all the response values are being pushed into an array
Inside the for loop I want the final array which has all the values of the api response. So, I check the total page value and response page Number if that matches which means that will be the last page and I push the array to another function but when I do that it does not have all the values because nodejs does not wait for the api response.
const fs = require('fs');
var pepKey = 'asdfasdfasd';
var pepResponse;
var pepTimecards = [];
pep();
function pep(){
var options = {
headers: {
"content-type": "application/json",
},
agentOptions: {
pfx: fs.readFileSync('./certificate/asdfsdaf.p12'),
passphrase: 'asdasdsda'
}
};
request.get('https://source.asdfasdf.io/api/organisations/asdfasdf/timecard_keys?timecard_type=Flex',options, (err, res, body) => {
if (err) { return console.log(err); }
pepResponse = JSON.parse(body)
pepTimecards = pepResponse.data;
if(pepResponse.pages > 1){
for(let i=2;i<=pepResponse.pages;i++){
var url = 'https://source.`pepme`.io/api/organisations/sdfsadf/timecard_keys?timecard_type=Flex&page='+pageNo;
request.get(url,options, (err, res, body) => {
if (err) { return console.log(err); }
body = JSON.parse(body)
pepTimecards = pepTimecards.concat(body.data)
if(pepResponse.pages == body.page){
console.log(pepResponse.pages)
console.log(body.page +"body page")
console.log(pepTimecards)
}
});
}
}else{
}
});
}

Answer 1

Use the request-promise library which supplies promisified versions of the request library. Then, you can use async/await in your for loop to serialize your operations:
Newer answer to go with the edited code in the OP's question
const fs = require('fs');
const rp = require('request-promise');
const pepKey = 'asdfasdfasd';
pep().then(pepTimecards => {
// the timecard data is valid in here
console.log(pepTimecards);
}).catch(err => {
console.log(err);
});
async function pep() {
let timecards = [];
const options = {
headers: {
"content-type": "application/json",
},
agentOptions: {
pfx: fs.readFileSync('./certificate/asdfsdaf.p12'),
passphrase: 'asdasdsda'
},
json: true,
uri: 'https://source.asdfasdf.io/api/organisations/asdfasdf/timecard_keys?timecard_type=Flex'
};
let pepResponse = await rp(options);
timecards = pepResponse.data;
if (pepResponse.pages > 1) {
for (let i = 2; i <= pepResponse.pages; i++) {
options.uri = 'https://source.`pepme`.io/api/organisations/sdfsadf/timecard_keys?timecard_type=Flex&page='+pageNo;
let body = await rp(url, options);
// add body.data onto the existing array
timecards.push(...body.data);
}
} else {
}
console.log(pepResponse.pages)
console.log(timecards)
return timecards;
}
Prior Answer before OP edited the code in their question:
const rp = require('request-promise');
// I'm assuming this is some sort of method definition on a class, otherwise it needs the function keyword
async pageno(pageNo) {
for (let i=2;i<=pepResponse.pages;i++){
try {
options.uri = 'https://test/timecard_keys?timecard_type=asdas&page='+pageNo;
// let request-promise parse the json for you automatically
options.json = true;
let body = await rp(options);
pepTimecards = pepTimecards.concat(body.data)
if (pepResponse.pages == body.page){
console.log(pepResponse.pages)
console.log(body.page +"body page")
console.log(pepTimecards)
}
} catch(e) {
// decide what to do for error handling
// this will log and rethrow so the caller will get a rejected promise
console.log(e);
throw e;
}
}
// return some value here to be the resolved value of the returned promise
return pepTimecards;
}
In your code, it is not clear where the options, pepTimecards, pepResponse variables are declared. They should probably be declared as local variables here or passed in to the function and/or returned from your function.
Summary of modifications:
Add async to method declaration so we can use await.
Load request-promise library into rp variable
Add options.json = true to the let the request-promise library parse the JSON result for us automatically
Change rp() to just use the options structure (add URL to that)
Add try/catch to catch any errors from the await, log them, then rethrow so pageno() will return a promise that rejects if there is an error (you can customize the behavior when there's an error if desired)
Add a return value so there is meaningful resolved value to the promise (you should not be using side-effect programming as it is now (modifying variables that are not passed in, declared locally or returned).
Things for you still to fix:
Stop using side-effect programming where you modify free variables that aren't passed in, aren't declared locally and aren't returned. This is a bad way to design code. You don't show enough overall context from the calling code or where these other variables are defined to make a concrete recommendation on how it should be done.
Decide what your error handling strategy is if there's an error on one of the requests and implement that strategy and proper handling.