node.js loop through array to write files - arrays

I've been working through a few others, also this of looping through array to http.get data from a variety of sources. I understand that nodeJS is working asynchronously which is allowing the files to be written empty or with incomplete data, but I can't seem to get past this point.
Problem: calls are made, files are built but the files are always empty
Goal: loop through an array to create files locally from the sites data. Here is what I've got so far:
var file_url = 'http://js.arcgis.com/3.8amd/js/esri/',
DOWNLOAD_DIR = './esri/',
esriAMD = [ '_coremap.js', 'arcgis/csv.js'];
function readFile(callback) {
if (esriAMD.length > 0) {
var setFile = esriAMD.shift(),
file_name = url.parse(file_url).pathname.split('/').pop(),
trial = setFile.split('/').pop(),
file = fs.createWriteStream(DOWNLOAD_DIR + trial);
http.get(file_url + esriAMD, function(res) {
res.on('data', function(data) {
file.write(data);
console.log(setFile + ' has been written successfully');
});
res.on('end', function(){
console.log(setFile + ' written, moving on');
console.log(esriAMD.length);
readFile(callback);
});
//readFile(callback);
});
} else {
callback();
}
}
readFile(function() {
console.log("reading finishes");
});
Any insight would really help.
thanks,

var esriAMD = [....];
...
function readFile(callback) {
...
http.get(file_url + esriAMD, function(res) {
...
concatenating strings with arrays may yield unexpected results.
you want to make sure that
you know what URLs your program is accessing
your program deals with error situations (where the fsck is res.on('error', ...)?)

Solution: I was passing the wrong variable into the http.get
Working code:
var file_url = 'http://.....',
DOWNLOAD_DIR = './location/';
esriAMD = ['one', 'two', 'three'..0;
function readFile(callback) {
if(esriAMD.length > 0) {
var setFile = esriAMD.shift(),
file_name = url.parse(setFile).pathname.split('/').pop(),
trial = setFile.split('/').pop(),
file = fs.createWriteStream(DOWNLOAD_DIR + trial);
http.get(file_url + setFile, function(res){
res.on('error', function(err){
console.log(err);
});
res.on('data', function(data){
file.write(data);
console.log(setFile + ' started');
});
res.on('end', function(){
console.log(setFile + ' completed, moving on');
});
});
} else {
callback();
}
}

Related

throw new ERR_INVALID_ARG_TYPE('chunk',['string','Buffer'],chunk);TypeError[ERR_INVALID_ARG_TYPE]:The "chunk" arg must be type string or Buffer

I am trying to get the contents of a .json file using a node js service into an angularjs method. But am getting following error:
_http_outgoing.js:700
throw new ERR_INVALID_ARG_TYPE('chunk', ['string', 'Buffer'], chunk);
^
TypeError [ERR_INVALID_ARG_TYPE]: The "chunk" argument must be one of type string or Buffer. Received type object
at ServerResponse.end (_http_outgoing.js:700:13)
here are the corresponding code fragments...
angular controller: the commented lines are all of those which i have tried and failed with.
var currentProcess = "process_1cA";
$scope.storestats = [];
var resAss = $resource('/procs/getstorestats');
var stats = resAss.get({
process: currentProcess,
date: date.getFullYear() + "" + m + "" + d
});
stats.$promise.then(function(response) {
if (response != undefined) {
// var r = JSON.parse(response);
//$scope.storestats.push(r);
//$scope.storestats.push(r);
//var r = JSON.parse(response);
$scope.storestats.push(response);
//angular.forEach(r, function(value, key) {
// $scope.storestats.push({key : value});
//});
}
});
NODEJs service:
httpApp.get('/procs/getstorestats', function(req, res, next) {
try {
fs.readFile(cfg.routestatspath + "storestats-"+req.query.process + "-" + req.query.date + ".json", function (err, data) {
var msgs1 = JSON.parse(data);
//var r = data.toString('utf8');
var msgs2 = JSON.stringify(msgs1);
console.log(msgs1);
res.end(msgs1);
});
}
catch (err) {
res.end(err.toString());
}});
P.S: The commented out lines are those which i have tried out with and failed. Also, the commented lines in the node service code snippet, give no error, and when logged show it correctly, but the data when in response of the controllers is blank.
I'm guessing a bit here, but I think you just need to change res.end() to res.send() in your Node code. The "end" method is used when you are streaming chunks of data and then you call end() when you're all done. The "send" method is for sending a response in one go and letting Node handle the streaming.
Also, be sure you are sending a string back!
httpApp.get('/procs/getstorestats', function(req, res, next) {
try {
fs.readFile(cfg.routestatspath + "storestats-"+req.query.process + "-" + req.query.date + ".json", function (err, data) {
var msgs1 = JSON.parse(data);
//var r = data.toString('utf8');
var msgs2 = JSON.stringify(msgs1);
console.log(msgs1);
res.send(msgs2); // NOTE THE CHANGE to `msg2` (the string version)
});
}
catch (err) {
res.send(err.toString()); // NOTE THE CHANGE
}
});
I had a similar error. It was because I was passing process.pid to res.end(). It worked when I changed process.pid to string
res.end(process.pid.toString());
Figured it out. 2 small changes were needed.. One in the controller, which was to use a "$resource.query" instead of "$resource.get". And in the service, as #jakarella said, had to use the stringified part in the .end();
Controller:
var resAss = $resource('/procs/getstorestats');
var stats = resAss.query({process: currentProcess, date: date.getFullYear() + "" + m + "" + d});
stats.$promise.then(function (response) {
$scope.storestats.push(response);
}
Node Service:
httpApp.get('/procs/getstorestats', function(req, res, next) {
try {
fs.readFile(cfg.routestatspath + "storestats-"+req.query.process + "-" + req.query.date + ".json", function (err, data) {
var msgs1 = JSON.parse(data);
var msgs2 = JSON.stringify(msgs1);
console.log(msgs2);
res.end(msgs2);
});
}
If you are using 'request-promise' library set the json
var options = {
uri: 'https://api.github.com/user/repos',
qs: {
access_token: 'xxxxx xxxxx'
},
headers: {
'User-Agent': 'Request-Promise'
},
json: true // Automatically parses the JSON string in the response
};
rp(options)
.then(function (repos) {
})
.catch(function (err) {
});
Thank you user6184932, it work
try {
await insertNewDocument(fileNameDB, taskId);
res.end(process.pid.toString());
} catch (error) {
console.log("error ocurred", error);
res.send({
"code": 400,
"failed": "error ocurred"
})
}
in mysql2 the reason for the error is the sql word , sql is a query :
const sql = select * from tableName
pool.executeQuery({
sql,
name: 'Error list for given SRC ID',
values: [],
errorMsg: 'Error occurred on fetching '
})
.then(data => {
res.status(200).json({ data })
})
.catch(err => {
console.log('\n \n == db , icorp fetching erro ====> : ', err.message, '\n \n')
})
I got the error using Node v12 (12.14.1).
Uncaught TypeError [ERR_INVALID_ARG_TYPE]: The "chunk" argument must be one of type string or Buffer. Received type number
Sample code for context.
const { Readable } = require('stream')
Readable.from(Buffer.from(base64content, 'base64'))
.pipe( ... )
Solution (for my case), was upgrading to Node v14 (14.17.3). e.g.
nvm use 14
nvm

Angular template won't load. Even with $loaded. Data resolves after Load

Using AngularFire, Angular, Firebase.
I load a list of users from a Firebase Database. I use $loaded to ensure it waits until data loads.
I take this list, compare it against another firebase database of groups and push the results into two arrays.
Based on the console.logs the data sorts correctly. However, inside my template I get a blank page (I think this is because the page loads before the data is sorted).
Thoughts?
let userLoggedIn = AuthFactory.getUser();
var allUsersArray = $firebaseArray(ConnectFactory.fbUserDb);
var x = firebase.database().ref('groups');
var friendArr = [];
var notFriendArr = [];
allUsersArray.$loaded().then(function(){
angular.forEach(allUsersArray, function(user, i) {
var haveIAdded = x.child(userLoggedIn).child(allUsersArray[i].uid).once('value').then(function (snap) {
if (snap.val() !== null) {
return true;
} else {
return false;
}
});
var haveTheyAdded = x.child(allUsersArray[i].uid).child(userLoggedIn).once('value').then(function (snap) {
if (snap.val() !== null) {
return true;
} else {
return false;
}
});
Promise.all([haveIAdded, haveTheyAdded]).then(function([you, they]) {
if (you && they) {
console.log('We Are Friends', allUsersArray[i]);
friendArr.push(allUsersArray[i]);
} else {
console.log('not a friend ', allUsersArray[i]);
notFriendArr.push(allUsersArray[i]);
}
});
});
$scope.friendList = friendArr;
$scope.notFriendList = notFriendArr;
});
Alright, this time I tried to actually read the question before attempting to answer. ;-)
When you set your $scope.friendList and $scope.notFriendList within the $loaded promise, your Promise.all may (and most likely) havn't resolved yet when those are called, since angular.forEach doesn't wait for the promises to finish before moving on to the next statement in the function. So you'll have to build an array of promises and wait for them all to resolve outside of the loop before attempting to set your $scope variables.
allUsersArray.$loaded().then(function(){
var promises = [];
var friendArr = [];
var notFriendArr = [];
angular.forEach(allUsersArray, function(user, i) {
... // Same as before
promises.push(
Promise.all([haveIAdded, haveTheyAdded]).then(function([you, they]) {
if (you && they) {
console.log('We Are Friends', allUsersArray[i]);
friendArr.push(allUsersArray[i]);
} else {
console.log('not a friend ', allUsersArray[i]);
notFriendArr.push(allUsersArray[i]);
}
})
);
});
Promise.all(promises).then(function(){
$scope.friendList = friendArr;
$scope.notFriendList = notFriendArr;
});
});

Angular datatable and JSON structure issue

Here is my issue:
The datatable plug-in only accepts an array I believe but my API returns an object with an array. I am trying to figure out if I need to extract this info (how?) or if the plug-in has some methods that do that for me (there is one):
//the datatable plugin expects the following JSON structure
[
{
"ID":"a5f415a7-3d4f-11e5-b52f-b82a72d52c35",
"Record":1,
"HostName":"SRX552P"
}
]
//my PHP server returns the following JSON structure:
{
"status":"success",
"message":"data retrieved",
"data":[
{
"ID":"a5f415a7-3d4f-11e5-b52f-b82a72d52c35",
"Record":1,
"HostName":"SRX552P"
}
]
}
var allData = null;
//this is my angularjs service where I am grabbing all the 'data':
function getAllData() {
dataservice.get('table001').then(function(data){
allData = data;
});
return allData;
}
//it looks like my issue is exactly what this post describes:
http://stackoverflow.com/questions/27797435/accessing-json-data-in-angularjs-datatables-using-dtoptions
//but applying ".withDataProp('data.data')" didn't work for me:
...
this.standardOptions = DTOptionsBuilder
.fromFnPromise(getAllData())
.withDataProp('data.data')
//.fromSource('api/tables/datatables.standard.json') //static data works fine!
//Add Bootstrap compatibility
.withDOM("<'dt-toolbar'<'col-xs-12 col-sm-6'f><'col-sm-6 col-xs-12 hidden-xs'l>r>" +
"t" +
"<'dt-toolbar-footer'<'col-sm-6 col-xs-12 hidden-xs'i><'col-xs-12 col-sm-6'p>>")
.withBootstrap();
this.standardColumns = [
DTColumnBuilder.newColumn('ID'),
DTColumnBuilder.newColumn('Record'),
DTColumnBuilder.newColumn('HostName')
];
...
//this is the service
(function () {
'use strict';
angular.module('app.ipxtool').factory('dataservice', dataservice);
dataservice.$inject = ['$http', '$q'];
function dataservice($http, $q) {
var serviceBase = 'api/v1/';
var obj = {};
obj.get = function (q) {
return $http.get(serviceBase + q).then(success).catch(fail);
};
function success(results) {
if (results.data.status == "error") {
logger.error(results.data.message, '', 'Error'); //$response["data"] = null for errors;
}
if (results.data.status == "warning") {
logger.warning(results.data.message, '', 'Warning');
}
if (results.data.status == "success") {
logger.success(results.data.message, results.data.data, 'Success'); //$response["data"] = $rows;
}
return results.data;
}
function fail(e) {
return (e);
}
return obj;
};
})();
Using Fiddler I can see all the data being returned. Also I output the first array item as follows:
console.log("var allData: " + "[" + JSON.stringify(alldata.data[1]) + "]");
The solution to this issue was to add the following:
.fromFnPromise(function() {
return dataservice.get('table001');
})
Not sure why calling getAllData() didn't work. So finally I got this thing resolved. Thanks.
You can add [ and ] at the start and at the end of your output var in php page itself, so that it would return a proper JSON output. Since I also faced such issue earlier in Angular as it doesn't accept direct Object input, I used the above approach to convert it into full-fledged JSON.
Also add the below code at the top of your PHP page:
header('Content-Type: application/json');
So that, it'll return the content in JSON-formatted text.

how to access /obtain object elements from an observable array

I would like to know how I can obtain an obtain an objects item(LicenseNo) from an observable array. I have pasted my code below , after inserting breakpoints I noticed that I am currently getting the entire objects array as in [LicenseNo , name , allocated route , licenseDate] I just want to get the LicenseNo
function getDriverList(item) {
if (!pageViewModel.isAuthenticated()) return;
var dfd = jQuery.Deferred();
var LicenseNo = ko.observableArray([]);
$.when(getSecureData("/api/FleetDrivers/" + item.NationalID))
.done(function (resp) {
pageViewModel.vehicelDriversVM.DriverList(resp.FleetDriverList);
if (pageViewModel.vehicelDriversVM.DriverList(resp.FleetDriverList).length > 0) {
LicenseNo = pageViewModel.vehicelDriversVM.FleetDriverList()[0];
alert('The first driver's License Number is ' LicenseNo;
}
});
return dfd.promise();
}
the code above didnt work so I have pasted the correct one below for anyone who might run into a similar prob
function getDriverList(item) {
if (!pageViewModel.isAuthenticated()) return;
var dfd = jQuery.Deferred();
var LicenseNo = ko.observableArray();
$.when(getSecureData("/api/FleetDrivers/" + item.NationalID))
.done(function (resp) {
pageViewModel.vehicelDriversVM.DriverList(resp.FleetDriverList);
LicenseNo = resp.FleetDriverList[1];
alert('The first driver's License Number is ' + LicenseNo);
});
return dfd.promise();
enter code here

AngularJS ng-repeat view not updating after data update

have a some data one my page that is in a ng-repeat.
When the page and data 1st loads the data shows up.
When I move away from the page (using Angular Routing) make a change to the data (gets saved in db) then come back into the page (make call to db get new data) the ng-repeat data does not refresh. I can see the new data loading into the array and it is the new data.
I start the process on the page with
var sp = this;
sp.viewData = [];
sp.employee = [];
sp.ViewDataTwo = [];
$(document).ready(function () {
var testHeader = setInterval(function () { myTimer() }, 1000);
function myTimer() {
if (addHeaderToken() != undefined) {
clearInterval(testHeader);
sp.usageText = "";
if (sessionStorage.getItem(tokenKey) != null) {
sp.associatedInfo = JSON.parse(getassociatedInfo());
loadDataOne();
loadDataTwo();
}
}
}
});
I do this because I need to get my security toke from a JS script that I have no power over changes. So I need to make sure the code has ran to get me the token.
here are the functions I call..
function loadPasses() {
$http.defaults.headers.common.Authorization = "Bearer " + addHeaderToken();
$http.get('/api/Employee/xxx', { params: { employeeId: sp.employeeId } }).then(function (data) {
sp.viewData = data.data;
for (var i = 0; i < $scope. viewData.length; i++) {
sp.passes[i].sortDateDisplay = (data.data.status == "Active" ? data.data.DateStart + "-" + data.data[i].DateEnd : data.data[i].visitDate);
sp.passes[i].sortDate = (data.data[i].status == "Active" ? data.data[i].DateStart: data.data[i].visitDate);
}
});
}
function loadDataTwo () {
$http.defaults.headers.common.Authorization = "Bearer " + addHeaderToken();
if (sessionStorage.getItem(tokenKey) != null) $http.get('/api/Employee',
{
params: { employeeId: sp.employeeId }
}).then(function (data) {
sp.employee = data.data;
var tempPassString = "";
sp.ViewDataTwo = [];
var totalA = 0;
var totalU = 0;
for (var p = 0; p < sp.employee.dataX.length; p++) {
sp.ViewDataTwo.push(sp.employee.dataX[p].description + "(" + /** math to update description **// + ")");
totalA += parseInt(parseInt(sp.employee.dataX[p].Anumber));
totalU += parseInt(sp.employee.dataX[p].Bnumber));
}
sp.usageArr.push(" Total: " + totalA- totalU) + "/" + totalA + " Available");
//$scope.$apply();
});
}
One my view sp.viewData and sp.ViewDataTwo are both in ng-repeats.
Works well on load.. when I go out and come back in. I see the data reloading. But the view does not.
I have hacked the Dom to get it to work for now. But I would like to do it the right way..
Any help.
I have used
$scope.$apply();
But it tells me the digest is already in process;
the views are in a template..
Please help

Resources