Nodejs S3 Delete Multiple Objects Error - arrays

I am trying to bulk delete my s3 objects that are associated with one specific blog record in my database, but I'm getting hung up on how to pass the array to my params object to be used in the s3.deleteObjects method, but I'm held up on this error: Check with error message InvalidParameterType: Expected params.Delete.Objects[0].Key to be a string. I feel like it could be related to not having a loop at some point in the process or maybe the format of the values being passed to my s3File array.
Here is the my routing:
.delete(function(req, res){
models.File.findAll({
where: {
blogId: blog.blogId
}
}).then(function(file){
var s3Files = [];
function s3Key(link){
var parsedUrl = url.parse(link);
var fileName = parsedUrl.path.substring(1);
return fileName;
}
for(var k in file){
console.log('Here are each files ' + file[k].fileName);
s3Files.push(s3Key(file[k].fileName));
}
console.log('Here are the s3Files ' + s3Files);
//GOTTEN TO THIS POINT WITHOUT AN ERROR
aws.config.update({accessKeyId: process.env.AWS_ACCESS_KEY, secretAccessKey: process.env.AWS_SECRET_KEY, region: process.env.AWS_REGION});
//var awsKeyPath = s3Key(file.fileName);
var s3 = new aws.S3();
var options = {
Bucket: process.env.AWS_BUCKET,
Delete: {
Objects: [{
Key: s3Files
}],
},
};
s3.deleteObjects(options, function(err, data){
if(data){
console.log("File successfully deleted");
} else {
console.log("Check with error message " + err);
}
});
});
Here is the output from console.log('Here are each files ' + file[k].fileName);:
Here are each files https://local-bucket.s3.amazonaws.com/1/2017-02-12/screen_shot_2017-02-01_at_8_25_03_pm.png
Here are each files https://local-bucket.s3.amazonaws.com/1/2017-02-13/test.xlsx
Here are each files https://local-bucket.s3.amazonaws.com/1/2017-02-13/screen-shot-2017-02-08-at-8.23.37-pm.png
Here is the output from console.log('Here are the s3Files ' + s3Files);:
Here are the s3Files 1/2017-02-12/screen_shot_2017-02-01_at_8_25_03_pm.png,1/2017-02-13/test.xlsx,1/2017-02-13/screen-shot-2017-02-08-at-8.23.37-pm.png
Here is the error message:
Check with error message InvalidParameterType: Expected params.Delete.Objects[0].Key to be a string

Key should be a string. You should use array of Object to Objects.
Use this code :
var objects = [];
for(var k in file){
objects.push({Key : file[k].fileName});
}
var options = {
Bucket: process.env.AWS_BUCKET,
Delete: {
Objects: objects
}
};

Change your array as an object
const objects = [
{Key: 'image1.jpg'},
{Key: 'image2.jpg'}
]
Add a new item to the list
for(var k in file){
objects.push({Key : file[k].fileName});
}
Set the array as Objects value in parameters
const options = {
Bucket: process.env.BUCKET,
Delete: {
Objects: objects,
Quiet: false
}
};
Now delete objects
s3.deleteObjects(options, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
Learn more from official docs

Related

How to retrieve multiple image from Amazon S3 using imgURL at once?

I want to retrieve list of images in one go from Amazon S3 based on image URL.
Currently I am able to fetch single image using the following code:-
AWS.config.update({
accessKeyId: accessKeyId,
secretAccessKey: secretAccessKey
});
AWS.config.region = region;
var bucketInstance = new AWS.S3();
var params = {
Bucket: bucketName,
Key: awsImgUrl
}
bucketInstance.getObject(params, function (err, file) {
if (file) {
var dataSrc = "data:" + file.ContentType + ";base64," + EncodeData(file.Body);
callbackSuccess(dataSrc);
} else {
callbackSuccess("Error");
}
});
EncodeData = function (data) {
var str = data.reduce(function (a, b) { return a + String.fromCharCode(b) }, '');
return btoa(str).replace(/.{76}(?=.)/g, '$&\n');
}
In my scenario I have multiple S3 image url like awsImgUrl1, awsImgUrl2..awsImgUrln.
How to fetch it in one go instead of one by one?
You cannot get more than one image per api call with S3. You can however make multiple calls in parallel.
Using promises this is straightforward.
var bucketInstance = new AWS.S3();
var imageKeys = [ awsImgUrl1, awsImgUrl2, awsImgUrl3];
var promisesOfS3Objects = imageKeys.map(function(key) {
return bucketInstance.getObject({
Bucket: bucketName,
Key: key
}).promise()
.then(function (file) {
return "data:" + file.ContentType + ";base64," + EncodeData(file.Body);
})
})
Promise.all(promisesOfS3Objects)
.then(callbackSuccess) // callbackSuccess is called with an array of string
.catch(function() { callbackSuccess("Error") })
You can change the way you upload the image data. Instead of uploading a single image, upload one document containing multiple image datas.
const addImageBlock = () => {
var photoBlock = [
{
imageId: 'id',
type: 'png',
body: 'data:image/png;base64,iVBORw0K...'
},
{
imageId: 'id2',
type: 'png',
body: 'data:image/png;base64,iVBORw0K...'
},
{
imageId: 'id3',
type: 'png',
body: 'data:image/png;base64,iVBORw0K...'
},
{
imageId: 'id4',
type: 'png',
body: 'data:image/png;base64,iVBORw0K...'
}
//...ect
];
s3.upload({
Key: photoBlockId + '.json',
Body: photoBlock,
ACL: 'public-read'
}, function(err, data) {
if (err) {
return alert('There was an error', err.message);
}
});
}
Then when you receive this data with one s3 call, you can loop through and render the images on the frontend,
getObject(params, function (err, file) {
imageArr = [];
if (file) {
JSON.parse(file.toString()).map((image) => {
var image = new Image();
image.src = image.body;
imageArr.push(image)
})
callbackSuccess(imageArr);
}
else {
callbackSuccess("Error");
}
});
AWS SDK does not have any method to read multiple files as once and same with console, you can not download multiple files at once.
they have only GetObject method do read a object in bucket by key only.
so in your case you have to read one by one with their key name only if you already have key names as list..
you can get summary of objects in bucket if you would like to get list of objects then put a loop to download all files.

throw new ERR_INVALID_ARG_TYPE('chunk',['string','Buffer'],chunk);TypeError[ERR_INVALID_ARG_TYPE]:The "chunk" arg must be type string or Buffer

I am trying to get the contents of a .json file using a node js service into an angularjs method. But am getting following error:
_http_outgoing.js:700
throw new ERR_INVALID_ARG_TYPE('chunk', ['string', 'Buffer'], chunk);
^
TypeError [ERR_INVALID_ARG_TYPE]: The "chunk" argument must be one of type string or Buffer. Received type object
at ServerResponse.end (_http_outgoing.js:700:13)
here are the corresponding code fragments...
angular controller: the commented lines are all of those which i have tried and failed with.
var currentProcess = "process_1cA";
$scope.storestats = [];
var resAss = $resource('/procs/getstorestats');
var stats = resAss.get({
process: currentProcess,
date: date.getFullYear() + "" + m + "" + d
});
stats.$promise.then(function(response) {
if (response != undefined) {
// var r = JSON.parse(response);
//$scope.storestats.push(r);
//$scope.storestats.push(r);
//var r = JSON.parse(response);
$scope.storestats.push(response);
//angular.forEach(r, function(value, key) {
// $scope.storestats.push({key : value});
//});
}
});
NODEJs service:
httpApp.get('/procs/getstorestats', function(req, res, next) {
try {
fs.readFile(cfg.routestatspath + "storestats-"+req.query.process + "-" + req.query.date + ".json", function (err, data) {
var msgs1 = JSON.parse(data);
//var r = data.toString('utf8');
var msgs2 = JSON.stringify(msgs1);
console.log(msgs1);
res.end(msgs1);
});
}
catch (err) {
res.end(err.toString());
}});
P.S: The commented out lines are those which i have tried out with and failed. Also, the commented lines in the node service code snippet, give no error, and when logged show it correctly, but the data when in response of the controllers is blank.
I'm guessing a bit here, but I think you just need to change res.end() to res.send() in your Node code. The "end" method is used when you are streaming chunks of data and then you call end() when you're all done. The "send" method is for sending a response in one go and letting Node handle the streaming.
Also, be sure you are sending a string back!
httpApp.get('/procs/getstorestats', function(req, res, next) {
try {
fs.readFile(cfg.routestatspath + "storestats-"+req.query.process + "-" + req.query.date + ".json", function (err, data) {
var msgs1 = JSON.parse(data);
//var r = data.toString('utf8');
var msgs2 = JSON.stringify(msgs1);
console.log(msgs1);
res.send(msgs2); // NOTE THE CHANGE to `msg2` (the string version)
});
}
catch (err) {
res.send(err.toString()); // NOTE THE CHANGE
}
});
I had a similar error. It was because I was passing process.pid to res.end(). It worked when I changed process.pid to string
res.end(process.pid.toString());
Figured it out. 2 small changes were needed.. One in the controller, which was to use a "$resource.query" instead of "$resource.get". And in the service, as #jakarella said, had to use the stringified part in the .end();
Controller:
var resAss = $resource('/procs/getstorestats');
var stats = resAss.query({process: currentProcess, date: date.getFullYear() + "" + m + "" + d});
stats.$promise.then(function (response) {
$scope.storestats.push(response);
}
Node Service:
httpApp.get('/procs/getstorestats', function(req, res, next) {
try {
fs.readFile(cfg.routestatspath + "storestats-"+req.query.process + "-" + req.query.date + ".json", function (err, data) {
var msgs1 = JSON.parse(data);
var msgs2 = JSON.stringify(msgs1);
console.log(msgs2);
res.end(msgs2);
});
}
If you are using 'request-promise' library set the json
var options = {
uri: 'https://api.github.com/user/repos',
qs: {
access_token: 'xxxxx xxxxx'
},
headers: {
'User-Agent': 'Request-Promise'
},
json: true // Automatically parses the JSON string in the response
};
rp(options)
.then(function (repos) {
})
.catch(function (err) {
});
Thank you user6184932, it work
try {
await insertNewDocument(fileNameDB, taskId);
res.end(process.pid.toString());
} catch (error) {
console.log("error ocurred", error);
res.send({
"code": 400,
"failed": "error ocurred"
})
}
in mysql2 the reason for the error is the sql word , sql is a query :
const sql = select * from tableName
pool.executeQuery({
sql,
name: 'Error list for given SRC ID',
values: [],
errorMsg: 'Error occurred on fetching '
})
.then(data => {
res.status(200).json({ data })
})
.catch(err => {
console.log('\n \n == db , icorp fetching erro ====> : ', err.message, '\n \n')
})
I got the error using Node v12 (12.14.1).
Uncaught TypeError [ERR_INVALID_ARG_TYPE]: The "chunk" argument must be one of type string or Buffer. Received type number
Sample code for context.
const { Readable } = require('stream')
Readable.from(Buffer.from(base64content, 'base64'))
.pipe( ... )
Solution (for my case), was upgrading to Node v14 (14.17.3). e.g.
nvm use 14
nvm

using tedious connection,need to get the total data

hai I am new to tedious and Es-6,It may be a silly question but I am struggling,
I want the total data in a array, using tedious connections here is my code:
getZipData() {
var Connection = require('tedious').Connection;
Request = require('tedious').Request;
var config = {
userName: 'xx',
password: 'xxxx',
server: 'xxx', // You can use 'localhost\\instance' to connect to named instance
options: {
database: 'xxxxx',
rowCollectionOnDone:'true'
}
}
var connection = new Connection(config);
var jsonArray = [];
connection.on('connect', function (err) {
if (err) {
console.log(err)
}
var sql = "SELECT * FROM xxxxx";
return new Promise(function(resolve,reject){
var request = new Request(sql,
(err, rowCount, rows)=>{
if (err) {
reject(err);
}
else {
alert("rows");
console.log(rowCount + 'rows');
}
});
request.on('row', (columns)=>{
var rowObject = {};
columns.forEach((column)=> {
rowObject[column.metadata.colName] = column.value;
});
jsonArray.push(rowObject);
});
connection.execSql(request);
request.on('done', function(rowCount, more) {
console.log(rowCount + ' rows returned');
alert("jsonArray2:"+jsonArray);
resolve(jsonArray)
});
});
})
}
componentWillMount() {
this.getZipData().then(function(resolved){
console.log(resolved);
alert("data:"+resolved);
}).catch(function(rejected){
console.log(rejected);
})
}
when i add the request.on('done', function(rowCount, more) also i didn't get any data can any one give the solution for it,
I want the total data to be displayed
It looks like you're calling resolve before your query has been executed:
var jsonArray = [];
// Register callback for row event
request.on('row', (columns)=>{
var rowObject = {};
columns.forEach((column)=> {
rowObject[column.metadata.colName] = column.value;
});
jsonArray.push(rowObject);
});
// Call resolve before executing request
resolve(jsonArray);
connection.execSql(request);
The docs mention a done event that indicates a request has completed:
request.on('done', function (rowCount, more, rows) {
// Call resolve here instead?
resolve(jsonArray);
});
Disclaimer: I've haven't actually used Tedious, but from the docs linked this looks like what you're looking for.

How to show multiple records fetched from a table in parse.com

I am using Parse for my app. I have two tables StudentDetail and Subject,as user enters his name, the id is queried from studentDetail table. and using that id i want to fetch rest of the details of Subject table.Data fetched through this code is correct but in $scope.subs i.e 19th line it overrides the data and returns only the last record,i want to store all the records in $scope.subs object iteratively.
$scope.subjectFetch= function(form,form1) {
var query = new Parse.Query(StudentDetail);
var querySub = new Parse.Query(Subject);
query.equalTo("Firstname",form1.Firstname);
query.find({
success: function(results) {
stdId = results[0].id;
querySub.equalTo("StudentId",stdId);
querySub.find({
success: function(subjects) {
alert("Success");
for (var i = 0; i < subjects.length; i++) {
var object = subjects[i];
subname = object.get('Name');
credits = object.get('credits');
code =object.get('code');
duration = object.get('duration');
alert("Subject name: "+subname+"\n Credits :"+credits+"\n Code :"+code+"\n Duration:"+duration);
$scope.subs=[{Name:subname,credits:credits,code:code,duration:duration},{Name:"xyz",credits:5}];
//console.log($scope.subs);
}
},
error: function(error) {
alert("Error: " + error.code + " " + error.message);
}
});
},
error: function(error) {
alert("Error: " + error.code + " " + error.message);
}
});
}
You are setting $scope.subs to be a new array with a single object in it inside your loop, which is why only the last one is there when it is done.
Try the following success handler instead:
success: function(subjects) {
// reset subs array
$scope.subs = [];
for (var i in subjects) {
var subject = subjects[i];
// push new object into the array
$scope.subs.push({
Name: subject.get('Name'),
credits: subject.get('credits'),
code: subject.get('code'),
duration: subject.get('duration')
});
}
// log the populated array
console.log($scope.subs);
Alternatively if you use something like the Underscore library, you can just map the items:
// replace success function body with this:
$scope.subs = _.map(subjects, function(subject) {
return {
Name: subject.get('Name'),
credits: subject.get('credits'),
code: subject.get('code'),
duration: subject.get('duration')
};
});
console.log($scope.subs);

Mongodb query doesn't work

In my image sharing application you can upload images and create albums. When you delete an image from the site it shall also be deleted in the albums (the ones that has got the image in it).
Below is the route for deleting an image, and what I really need help with is why the code for deleting the images (imageName and imageId) in the albums below doesn't work.
Thanks in advance!
The models:
var AlbumSchema = new Schema({
title : String,
imageName : [String],
imageId : [String]
});
modelObject.AlbumSchema = AlbumSchema;
modelObject.Album = mongoose.model('Album', AlbumSchema);
-
var BlogPostSchema = new Schema({
name : String,
size : Number,
type : String,
author : ObjectId,
title : String
});
modelObject.Comment = mongoose.model('Comment', CommentSchema);
modelObject.BlogPost = mongoose.model('BlogPost', BlogPostSchema);
The part that doesn't work in the code below is the following:
albums[i].imageName.remove(j);
albums[i].imageId.remove(j);
albums[i].save(function (err){
if (err) {
console.log(err);
// do something
}
});
Full code:
app.get('/blog/delete/:id', function(req, res){
model.BlogPost.findById(req.params.id, function (err, blog){
var theImage = blog.name;
var query = albumModel.Album.find( { imageName:theImage } )
query.exec(function (err, albums) {
if (!albums) {
blog.remove(function(err) {
console.log(err);
// do something
});
res.redirect('/blogs');
}
else {
for (var i = 0; i < albums.length; i++) {
for (var j = 0; j< albums[i].imageName.length; j++){
if (theImage == albums[i].imageName[j]){
albums[i].imageName.remove(j);
albums[i].imageId.remove(j);
albums[i].save(function (err){
if (err) {
console.log(err);
// do something
}
});
}
}
}
}
blog.remove(function(err) {
console.log(err);
// do something
});
res.redirect('/blogs');
});
});
});
JavaScript arrays don't have a remove method so I would expect your code may be crashing. You should be using code like albums[i].imageName.splice(j, 1); instead.

Resources