Using Node Buffer or fileStream with formData file upload - angularjs

Update: I have more or less solved the problem using multipart (app.use(multipart({uploadDir: __dirname + '/../uploads'}))from these instructions), but still don't know why my original code (below) fails.
There have been numerous variations on this question, and I have tried the ideas there without success. I'm using a file uploading directive (and have since tried another open source alternative) to send a binary file to a node server, that runs the following code (based on an SO answer I can't now refind):
exports.receive = function(req, res) {
var fitFileBuffer = new Buffer('');
// req.setEncoding("binary"); //doesn't help
req.on('data', function(chunk) {
fitFileBuffer = Buffer.concat([fitFileBuffer, chunk]);
});
req.on('end', function() {
fs.writeFileSync(
"today2.fit",
fitFileBuffer,
'binary');
res.send(200);
});
};
If I upload today.fit and compare to today2.fit, they have the same Kb of data, but are not identical, and subsequent code fails to process the file. Given that this happens with two pieces of third party code I suspect the problem lies with my code.
Here are the details from the client side of the POST being made

In the end, when I realised that I wanted to avoid saving to disk, I modified generalhenry's code with some stuff from busyboy's site and my own use of a Buffer:
exports.receive = function (req, res, next) {
var busboy = new Busboy({ headers: req.headers });
var fileBuffer = new Buffer('');
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
file.on('data', function(data) {
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
fileBuffer = Buffer.concat([fileBuffer, data]);
});
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
genXmlFromString(fileBuffer.toString(), function(data) {
res.json(data);
});
});
});
busboy.on('finish', function() {
console.log("'finish'");
});
req.pipe(busboy);
};

UPDATE: the client post details helped. You're not posting a file stream (which would have worked) you're posting a form stream. The good news is there are good modules for handling form streams.
You'll need to pipe the request stream into a form handling stream (such as busboy) which will handle the ------WebKitFormBoundary. . . part and them give you the file(s) as stream(s)
https://github.com/mscdex/busboy
var Busboy = require('busboy');
exports.receive = function(req, res, next) {
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
var fileWriteStream = fs.createWriteStream('today2.fit');
file.pipe(fileWriteStream);
});
busbody.on('finish', function() {
res.send(201);
});
req.pipe(busboy);
};

Related

passing extra info to multer Upload API

I am using the multer way for upload in node.js, explained in detail here https://ciphertrick.com/2015/12/07/file-upload-with-angularjs-and-nodejs/
I am trying to pass another information with requested data which is called invoiceId:
Upload.upload({
url: 'http://localhost:4000/api/ubiq/listInvoiceAttachedFiles/attach', //webAPI exposed to upload the file
data: {file: file, invoiceId:invoiceId} //pass file as data, should be user ng-model
}).then(function (resp) { //upload function returns a promise
if (resp.data.error_code === 0) { //validate success
$window.alert('Success ' + resp.config.data.file.name + ' uploaded');
console.log(resp.config.data.file); ....etc
But I am getting req.body empty on the server side:
/** API path that will upload the files */
server.post('/api/ubiq/listInvoiceAttachedFiles/attach', function(req, res) {
console.log(req.body);
security.verifyPermission("/api/ubiq/listInvoiceAttachedFiles/attach", req.session.currentUser, true /*isInSession*/).then(function (successInfo) {
if (!successInfo.isAllowed) {
console.log('not allowed');
return res.json(apiHelp.notAllowed());
}
What am I doing wrong?
Multer re-attaches the body object within the callback of the upload function.
Your data should be available after this line.
https://github.com/rahil471/file-upload-with-angularjs-and-nodejs/blob/master/server/app.js#L35
The following snippet is my working code and I have tested and used multiple time in my application.
Multer have two storage engines:
DiskStorage
MemoryStorage
I have used DiskStorage, which gives more control over disk storage on file.
var express = require("express");
var app = express()
var router = express.Router();
var multer = require("multer");
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, dirPath);
},
filename: function (req, file, cb) {
var datetimestamp = Date.now() + Math.floor(Math.random() * (1 - 99999999999 + 1)) + 9999999999999;
cb(null, datetimestamp + '.' + file.originalname.split('.')[file.originalname.split('.').length - 1].toLowerCase());
}
});
var infoUpload = multer({storage: storage});
router
.route(API_PATH)
.post(infoUpload.array("file"), function (req, res) {
console.log(req.body);
});
Following are few links which give you a brief explanation.
http://derpturkey.com/node-multipart-form-data-explained/
http://alexkatz.me/posts/image-upload-with-node-and-multer/
Hope it may help to resolve an issue.

NodeJS & Angular Image upload

Tried lot of available Internet tutorials, but I still can't make it functional. I use multer module in NodeJS and ng-file-upload in AngularJS. I made two helpers with multer settings( because I have two scenarios and both uploads must go to different folders). Backend files are in APP/APP_BACK/ folder, so in destination path I go back one folder and enter APP_FRONT/images/header. Here is one helper snippet (/helpers/uploadHeader.js):
var storage = multer.diskStorage({
destination: function (req, file, callback) {
callback(null, '../APP_FRONT/images/header/');
},
filename: function (req, file, callback) {
var ext = filename.slice((filename.lastIndexOf(".") - 1 >>> 0) + 2);
callback(null, file.fieldname + '-' + '.' + ext);
}
});
var upload = multer(
{storage: storage}
);
var helper = {
upload: upload
};
module.exports = helper;
Here is router file:
var headerHelper= require('../helpers/uploadHeader');
router.post('/header', headerHelper.upload.single('header'), function(req, res, next) {
headerHelper.upload(req, res, function(err){
if(err){
res.status(400).send(err);
console.log(err);
return;
}
res.status(200).send({ code:200, message:'Header upload successful!' });
});
});
"header" would be name of input form or key value in Postman.
In Angular, I injected 'ngFileUpload' module in app, and injected 'Upload' service into desired controller and used it inside uploadHeader() function which is bound on button inside form on clientside:
$scope.uploadHeader = function (file) {
Upload.upload({
url: 'http://localhost:3003/upload/header',
method: 'POST',
file: file
}).then(function (response) {
console.log('Success ' + response.config.data.file.name + 'uploaded. Response: ' + response.data);
}, function (error) {
console.log('Error status: ' + error.status);
}, function (evt) {
var progressPercentage = parseInt(100.0 * evt.loaded / evt.total);
console.log('progress: ' + progressPercentage + '% ' + evt.config.data.file.name);
});
I tried with Postman:
Postman request SS
And get this error:
"Error: ENOENT: no such file or directory, open 'c:\Users\Username\Desktop\APP\APP_FRONT\images\header\header.jpg'"
When I try from clientside, I get these errors:
"Error: Unexpected field at makeError "
and
"TypeError: dbg is undefined"
I already consulted Google, tried some of tutorials but got stuck on this.
In your router, you are telling multer to look for an object called "header" in the mulitpart form data. Your angularJS code is adding it as an object called 'file'.
If you change the line in your router from:
router.post('/header', headerHelper.upload.single(**'header'**), function(req, res, next) {
//.....
}
to:
router.post('/header', headerHelper.upload.single(**'file'**), function(req, res, next) {
//....
}
It should do the trick.
EDIT: solution found
Actually, it was problem in filename function in multer's storate settings. In above post, I appended datetimestamp and file extension to fieldname:
filename: function (req, file, callback) {
var datetimestamp = moment().format('DD-MM-YY_HH:mm:ss');
var filename = file.originalname;
var ext = filename.slice((filename.lastIndexOf(".") - 1 >>> 0) + 2);
callback(null, file.fieldname + '-' + datetimestamp + '.' + ext);
}
For some reason, it couldn't finish upload and I got error.
I changed filename function to return only file's originalname :
filename: function (req, file, callback) {
callback(null, file.originalname);
}
and now everything is OK. But now, question is, why custom filename doesn't work?

nodejs write simple image blob - Upload.dataUrltoBlob

Simple question. How do I save a image blob in Nodejs from angular.
AngularSide:
$scope.upload = function (dataUrl, picFile) {
Upload.upload({
url: 'http://test.dev:3000/register/user/uploads',
data: {
file: Upload.dataUrltoBlob(dataUrl, picFile.name)
},
}).then(function (response) {
$timeout(function () {
$scope.result = response.data;
});
}, function (response) {
if (response.status > 0) $scope.errorMsg = response.status
+ ': ' + response.data;
}, function (evt) {
$scope.progress = parseInt(100.0 * evt.loaded / evt.total);
});
}
nodejs side: Do I need middleware here? if so which one should I use?
router.post('/user/uploads', multipartMiddleware, function(req, resp) {
var newPath = "/Users/testUser/test_hold_files/" + req.files.file.originalFilename;
fs.writeFile(newPath, req.files.file, function(err) {
if (err) {
console.log("Data Error ");
return console.error(err);
}
});
res.status(200).jsonp({status: "status: success "});
});
right now this just writes out the file with correct name but its empty.
You used to be able to access the uploaded file through req.files.imageName and then you would fs.readFile from tmp and write it permanently, which is no longer the case in express 4.0
In Express 4, req.files is no longer available on the req object by default. To access uploaded files on the req.files object, use multipart-handling middleware like busboy, multer, formidable, multiparty, connect-multiparty, or pez.
Soooooooo, you can feel free to use which ever one of those middlewares names above and then follow their API for dealing with uploaded files like images. Hope this helps, enjoy.
Ok,
After a long time of messing with this stuff. I found an answer. It does load the file in my folder.
I feel this is only partial since it does not resize the actual file smaller. It is what is selected with https://github.com/danialfarid/ng-file-upload. I used the
Upload.upload({
url: 'http://test.dev:3000/register/user/uploads',
data: {
file: Upload.dataUrltoBlob(dataUrl, picFile.name)
},
This did zoom into the file on selected image. It did not make the actual file size smaller. I am still looking into this issue.
var formidable = require('formidable'),
util = require('util'),
fs_extra = require('fs-extra');
This is my post to accept images.
router.post('/user/uploads', function (req, res){
var form = new formidable.IncomingForm();
form.parse(req, function(err, fields, files) {
res.writeHead(200, {'content-type': 'text/plain'});
res.write('received upload:\n\n');
res.end(util.inspect({fields: fields, files: files}));
});
form.on('end', function(fields, files) {
/* Temporary location of our uploaded file */
var temp_path = this.openedFiles[0].path;
/* The file name of the uploaded file */
var file_name = this.openedFiles[0].name;
/* Location where we want to copy the uploaded file */
var new_location = "/Users/testUser/test_hold_files/";
fs_extra.copy(temp_path, new_location + file_name, function(err) {
if (err) {
console.error(err);
} else {
console.log("success!")
}
});
});
});
I have also noticed that I can view the file in chrome but not load it into gimp. Gimp gives me a file error.
Small steps I guess.
Maybe Datsik can give us some insight on what is going on here.
https://stackoverflow.com/users/2128168/datsik
Phil

Downloading a file from MongoDB to local filesystem

I have asked various questions and got only limited answers and it has gotten me this far:
Mongoose code :
app.get('/Download/:file(*)', function (req, res) {
grid.mongo = mongoose.mongo;
var gfs = grid(conn.db);
var file = req.params.file
var fs_write_stream = fs.createWriteStream('Program.cs');
var readstream = gfs.createReadStream({ filename: file });
readstream.pipe(fs_write_stream);
res.download(__dirname+'/Program.cs', 'Solution.cs', function (err) {
if (err) throw err;
else console.log("check console");
});
})
Angular code:
$scope.Download = function () {
$http.get(url + "/Download/"+"Program.cs")
.success(function (res) {
console.log(res);
})}
When I open my console angular just returns a blank line, when i look into the network aspect the header sais this:
I want to be able to store this file to my local file system to a path that I want to be able to choose myself. PLEASE help me out, I'm desperate, I have looked EVERYwhere for a solution, thank you very much in advance!
Try this instead of your client side download attempt:
<a class="btn" href="/Download/Program.cs" download>

Downloading a file from Mongoose in Angular

I've managed to upload a file and store it in my MongoDB, but now I want to be able to downlaod this file from the same mongoDB. In server-side I'm using the GridFS module in Mongoose to upload and download using the gfs-read/write-stream.
Downlaod code in Mongoose looks like :
app.post('/Download', function (req, res) {
grid.mongo = mongoose.mongo;
var gfs = grid(conn.db);
var readstream = gfs.createReadStream({
filename: 'Program.cs'
});
readstream.pipe(res);
})
In my angular i have this so far:
$scope.Download = function () {
$http.post(url + "/Download")
.success(function (res) {
console.log(res);
})
}
the console.log response is shown here
I want to save this content into a .cs file in my local file system, also want to be able to prompt the user for the download-path, How do I do this?
Try below code
app.post('/Download', function (req, res) {
var filenameId = "";// mention _id value of 'Program.cs'
var filename = 'Program.cs';
var _id = new ObjectID(filenameId );
var gfs = new Grid(conn.db, "yourfile_collection_name");// default value is fs
gfs.get(_id, function(err, data) {
if (err)
throw err;
res.setHeader('Content-Disposition','attachment; filename="' + filename + '"');
res.send(data);
});
};

Resources