Saving Location of Scraped Image to DB - Node/MEAN - angularjs

After scraping an image I'm able to download to a folder using request. I would like to pass along the location of this image to my Mongoose collection.
In the callback I think there should be a way to save the location so I can pass this along when saving my model object.
exports.createLook = function(req, res) {
var url = req.body.image;
var randomizer = '123456';
var download = function(url, filename, callback) {
request(url)
.pipe(fs.createWriteStream(filename))
.on('close', callback);
};
download(url, '../client/assets/images/' + randomizer + '.jpg', function() {
console.log('done');
// do something?
});
// now get model details to save
var newLook = new Look();
newLook.title = req.body.title;
newLook.image = // image location
newLook.save(function(err, look) {
if(err) return res.send(500);
} else {
res.send(item);
}
}

Assuming that 'randomizer' will be generated I would do:
exports.createLook = function(req, res) {
var url = req.body.image;
var randomizer = getSomthingRandom();
var download = function(url, filename, callback) {
request(url)
.pipe(fs.createWriteStream(filename))
.on('close', callback(filename);
};
download(url, '../client/assets/images/' + randomizer + '.jpg', function(filename) {
console.log('done');
// now get model details to save
var newLook = new Look();
newLook.title = req.body.title;
newLook.image = filename;
....
});

Related

how to export data into CSV and PDF files using angularjs

I want to, when i click on button (separate for both CSV and PDF), it automatically download in CSV and PDF file with correct Formatting.
this CSV code i want to add PDF inside code
$scope.downloadData = function() {
var datasets = $scope.datasets.reverse();
var file_name = $scope.m_id+ '.csv';
var dataUrl = 'data:text/csv;charset=utf-8,';
var json = [];
if(datasets !== null) {
for(idx = 0; idx < datasets.length; idx++) {
var dataset = datasets[idx].data;
var time = datasets[idx].timestamp;
time = $filter('date')(time, "dd/MMMM/yyyy-hh:mm a");
dataset.time = time;
json.push(dataset);
}
var fields = Object.keys(json[0]);
var csv = json.map(
function(row) {
return fields.map(
function(fieldName) {
return '"' + (row[fieldName] || '') + '"';
}
);
}
);
csv.unshift(fields);
var csv_str = csv.join('%0A');
var downloadURL = dataUrl + csv_str;
var saveAs = function(uri, filename) {
var link = document.createElement('a');
if (typeof link.download === 'string') {
document.body.appendChild(link); // Firefox requires the link to be in the body
link.download = filename;
link.href = uri;
link.target = "_blank";
link.click();
document.body.removeChild(link); // remove the link when done
} else {
location.replace(uri);
}
};
saveAs(downloadURL, file_name);
} else {
$scope.err_msg = 'Failed to get data. Try reloading the page.';
}
};
I try some of script i found on internet, but it is not working, some have formatting issue and save have downloading.
In Advance Thanks.
You should use this awesome library for pdf/csv or whatever else formats.. File Saver
Here's is code example, service created using FileSaver
function download(api, file, contentType) {
var d = $q.defer();
$http({
method: 'GET',
url: api,
responseType: 'arraybuffer',
headers: {
'Content-type': contentType
}
}).success(function(response) {
var data = new Blob([response], {
type: contentType+ ';charset=utf-8'
});
FileSaver.saveAs(data, file);
d.resolve(response);
}).error(function(response) {
d.reject(response);
});
return d.promise;
}
file input is name of file, you can use same service and pass the types and file names direct from controller.
Let;s you service name is homeService
for pdf call
homeservice.download('/api/download/whaever', 'export.pdf', 'application/pdf')

How to upload a file using filesystem I/O in Mean app?

I am using fs for uploading a file in my web app but the console shows that the file has been saved to the desired location which I have entered but the file doesn't show up there.
The code is here:-
var fs = require('fs-extra');
var path = require('path');
module.exports.updatePhoto = function(req,res) {
var file = req.files.file;
var userId = req.body.userId;
console.log("User "+ userId +" is submitting ", file);
var uploadDate = new Date();
var tempPath = file.path;
var targetPath = path.join(__dirname, "../../uploads/" + userId + uploadDate +file.name);
var savePath = "/uploads/" + userId + uploadDate + file.name;
fs.rename(tempPath, targetPath,function(err){
if(err) {
console.log(err);
}
else {
User.findById(userId, function(err, userData){
var user = userData;
user.image = savePath;
user.save(function(err){
if(err) {
console.log("failed")
res.json({status: 500})
}
else {
console.log("saved");
res.json({status: 200})
}
})
})
}
})
};
Are you using connect-multiparty to get the file from Express? https://github.com/expressjs/connect-multiparty
I ended up loading the files to AWS. So the best I can offer is the code to do that. It is basically free for my usage and I use docker to rebuild my site so this make it more flexible.
My File.Js:
'use strict';
module.exports = function (app) {
/**
* Module dependencies.
*/
var auth = require('../config/auth'),
api = {},
multiparty = require('connect-multiparty'),
multipartyMiddleware = multiparty(),
path = require('path'),
uuid = require('node-uuid'),
fs = require('fs'),
S3FS = require('s3fs');
var s3fsImpl = new S3FS('FOLDER_NAME_ENTER_HERE', {
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
});
/**
* Saves the logo file to the server
*/
api.uploadImage = function(req, res) {
// We are able to access req.files.file thanks to the multiparty middleware
var folder = req.params.folder;
var file = req.files.file;
var filename = uuid.v4() + path.extname(file.name);
var stream = fs.createReadStream(file.path);
s3fsImpl.writeFile(folder + '/' + filename, stream).then(function () {
fs.unlink(file.path, function (err) {
if (err) {
console.error(err);
}
});
return res.status(200).json({'fileName': filename, 'url': 'https://s3-us-west-2.amazonaws.com/AWS_FOLDER_ENTER_HERE' + folder + '/' + filename});
});
};
/**
* Routes
*/
app.route('/api/files/:folder/uploadImage')
.post(auth.jwtCheck, multipartyMiddleware, api.uploadImage);
};

Zipping multiple files in Nodejs having size ~ 300kb each and streaming to client

My code is working fine when I zip 3 files around 300kb each and send it to client. Used following links for help:
Dynamically create and stream zip to client
how to convert multiple files to compressed zip file using node js
But as soon as I try to zip 4th file I get "download - Failed Network error" in chrome.
Following is my code:
var express = require('express');
var app = express();
var fileSystem = require('fs');
var Archiver = require('archiver');
var util = require('util');
var AdmZip = require('adm-zip');
var config = require('./config');
var log_file = fileSystem.createWriteStream(__dirname + '/debug.log', {flags : 'a'});
logError = function(d) { //
log_file.write('[' + new Date().toUTCString() + '] ' + util.format(d) + '\n');
};
app.get('/zip', function(req, res, next) {
try {
res = setHeaderOfRes(res);
sendZip(req, res);
}catch (err) {
logError(err.message);
next(err); // This will call the error middleware for 500 error
}
});
var setHeaderOfRes = function (res){
res.setHeader("Access-Control-Allow-Origin", "*"); //Remove this when this is on production
res.setHeader("Content-Type", "application/zip");
res.setHeader("Content-disposition", "attachment;");
return res;
};
var sendZip = function (req, res) {
var filesNotFound = [];
zip.pipe(res);
if (req.query.leapIds) {
var leapIdsArray = req.query.leapIds.split(',');
var i, lengthi;
for (i = 0, lengthi = leapIdsArray.length; i < lengthi; i++) {
try {
var t = config.web.sharedFilePath + leapIdsArray[i] + '.vsdx';
if (fileSystem.statSync(t).isFile()) {
zip.append(new fileSystem.createReadStream(t), {
name: leapIdsArray[i] + '.vsdx'
});
};
} catch (err) {
filesNotFound.push(leapIdsArray[i] + '.vsdx');
}
}
var k, lengthk;
var str = '';
for (k = 0, lengthk = filesNotFound.length; k < lengthk; k++) {
str += filesNotFound[k] +',';
}
if(filesNotFound.length > 0){
zip.append('These file/files does not exist on server - ' + str , { name: 'logFile.log' });
}
zip.finalize();
}
};
I tried zip.file instead of zip.append that didn't work.
I want to zip minimum 10 files of 300kb each and send it to the client. Can anyone please let me know the approach.
Thanks
/********************* Update ****************************************
I was only looking at server.js created in node. Actually the data is sent correctly to client. Angularjs client code seems to be not working for large files.
$http.get(env.nodeJsServerUrl + "zip?leapIds=" + nodeDetails, { responseType: "arraybuffer" }
).then(function (response) {
nodesDetails = response.data;
var base64String = _arrayBufferToBase64(nodesDetails);
function _arrayBufferToBase64(buffer) {
var binary = '';
var bytes = new Uint8Array(buffer);
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary);
}
var anchor = angular.element('<a/>');
anchor.attr({
href: 'data:application/zip;base64,' + base64String,
target: '_blank',
download: $scope.main.routeParams.sectorId + "-ProcessFiles.zip"
})[0].click();
});
This part href: 'data:application/zip;base64,' + base64String, seems to be failing for large data received from server. For small files it is working. Large files it is failing.
Found out.
The problem was not in nodejs zipping logic. That worked perfect.
Issue was in the way I was handling the received response data.
If the data that is received is too large then following code fails
anchor.attr({
href: 'data:application/zip;base64,' + base64String,
target: '_blank',
download: $scope.main.routeParams.sectorId + "-ProcessFiles.zip"
})[0].click();
so the work around is to use blob:
function b64toBlob(b64Data, contentType, sliceSize) {
contentType = contentType || '';
sliceSize = sliceSize || 512;
var byteCharacters = atob(b64Data);
var byteArrays = [];
for (var offset = 0; offset < byteCharacters.length; offset += sliceSize) {
var slice = byteCharacters.slice(offset, offset + sliceSize);
var byteNumbers = new Array(slice.length);
for (var i = 0; i < slice.length; i++) {
byteNumbers[i] = slice.charCodeAt(i);
}
var byteArray = new Uint8Array(byteNumbers);
byteArrays.push(byteArray);
}
var blob = new Blob(byteArrays, { type: contentType });
return blob;
}
var contentType = 'application/zip'
var blob = b64toBlob(base64String, contentType);
saveAs(blob, "hello world.zip");
This link helped me out: How to save binary data of zip file in Javascript?
already answered here: https://stackoverflow.com/a/62639710/8612027
Sending a zip file as binary data with expressjs and node-zip:
app.get("/multipleinzip", (req, res) => {
var zip = new require('node-zip')();
var csv1 = "a,b,c,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
zip.file('test1.file', csv1);
var csv2 = "z,w,x,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
zip.file('test2.file', csv2);
var csv3 = "q,w,e,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
zip.file('test3.file', csv3);
var csv4 = "t,y,u,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
zip.file('test4.file', csv4);
var data = zip.generate({base64:false,compression:'DEFLATE'});
console.log(data); // ugly data
res.type("zip")
res.send(new Buffer(data, 'binary'));
})
Creating a download link for the zip file. Fetch data and convert the response to an arraybuffer with ->
//get the response from fetch as arrayBuffer...
var data = response.arrayBuffer();
const blob = new Blob([data]);
const fileName = `${filename}.${extension}`;
if (navigator.msSaveBlob) {
// IE 10+
navigator.msSaveBlob(blob, fileName);
} else {
const link = document.createElement('a');
// Browsers that support HTML5 download attribute
if (link.download !== undefined) {
const url = URL.createObjectURL(blob);
link.setAttribute('href', url);
link.setAttribute('download', fileName);
link.style.visibility = 'hidden';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}
}

Angular ng-file-upload Get Byte Array and FormData in Asp.net MVC

I am trying to use ng-file-upload to upload files using Angular. I need the byte array to store in our database (I cannot store the uploaded file on the server), but I also need the FormData as well. My problem is that I can only seem to get one or the other (either the byte array or the formdata) but not both.
Here is my Angular code:
$scope.uploadPic = function (file) {
$scope.emrDetailID = 7;
file.upload = Upload.upload({
url: '/SSQV4/SSQV5/api/Document/UploadEMRDocument',
method: 'POST',
data: { file: file, 'emrdetail': $scope.emrDetailID}
});
file.upload.then(function (response) {
$timeout(function () {
file.result = response.data;
$scope.imageID = file.result;
});
});
};
Using the code below, I can get the byte array and store it in my database:
public async Task<IHttpActionResult> UploadDocument()
{
var provider = new MultipartMemoryStreamProvider();
await Request.Content.ReadAsMultipartAsync(provider);
var f = provider.Contents.First(); // assumes that the file is the only data
if (f != null)
{
string ClientIP = IPNetworking.GetIP4Address();
var filename = f.Headers.ContentDisposition.FileName.Trim('\"');
filename = Path.GetFileName(filename);
var extension = Path.GetExtension(filename).TrimStart('.');
var buffer = await f.ReadAsByteArrayAsync();
FileImageParameterModel pm = new FileImageParameterModel();
pm.binFileImage = buffer;
//pm.CompanyID = UserInfo.intMajorID;
pm.CompanyID = 10707;
pm.dteDocumentDate = Convert.ToDateTime("4/4/2016");
pm.dteExpiration = Convert.ToDateTime("4/4/2017");
pm.vchUserIP = ClientIP;
pm.vchUploadedbyUserName = UserInfo.Username;
pm.vchFileExtension = extension;
CommonClient = new CommonWebApiClient();
CommonClient.AuthorizationToken = UserInfo.AccessToken;
int imageID = await CommonClient.InsertNewFileImage(pm);
return Json(imageID);
}
else
{
return BadRequest("Attachment failed to upload");
}
}
Using the code below I can get the FormData
var provider = new MultipartFormDataStreamProvider(workingFolder);
await Request.Content.ReadAsMultipartAsync(provider);
var emr = provider.FormData["emrdetail"];
but then I can't get the byte array as using MultipartFormDataStreamProvider wants a folder to store the file.
There's got to be a way to get both. I have been searching the internet for 2 days and all I can find are the two solutions above neither of which solves my issue.
Any assistance is greatly appreciated!
You are thinking way to complicated. Here is some of my code which I used for file upload in AngularJS with .NET
Angular:
function uploadFileToUrl(file) {
var formData = new FormData(); // Notice the FormData!!!
formData.append('uploadedFile', file);
return $http({
url: uploadUrl,
method: 'POST',
data: formData,
headers: {
'Content-Type': undefined
}
}).then(resolve, reject);
function resolve(data) {
$log.debug('data : ', data);
return data;
}
function reject(e) {
$log.warn('error in uploadFileToUrl : ', e);
return $q.reject(e);
}
}
Server:
public Task HandleAsync([NotNull] UploadFilesCommand command)
{
return wrapper.InvokeOnChannel(async client =>
{
// init command
command.Output = new Dictionary<string, int>();
try
{
foreach (var file in command.Files)
{
var request = new UploadFileRequest
{
FileName = file.Name,
FileStream = file.Stream
};
UploadFileResponse response = await client.UploadFileAsync(request);
command.Output.Add(file.Name, response.Id);
}
}
finally
{
// dispose streams
foreach (var file in command.Files)
{
if (file.Stream != null)
{
file.Stream.Dispose();
}
}
}
});
}

Uploading blob file to Amazon s3

I am using ngCropImage to crop an image and want to upload it following this link:
NgCropImage directive is returning me dataURI of the image and I am converting it to a blob (after converting it I get a blob object: which has size and type), Converted DataURI to blob using following code:
/*html*/
<img-crop image="myImage" result-image="myCroppedImage" result-image-size="250"></img-crop>
$scope.myImage='';
$scope.myCroppedImage = {image: ''}
var blob;
//called when user crops
var handleFileSelect=function(evt) {
var file=evt.currentTarget.files[0];
var reader = new FileReader();
reader.onload = function (evt) {
$scope.$apply(function($scope){
$scope.myImage=evt.target.result;
});
};
console.log($scope.myCroppedImage)
reader.readAsDataURL(file);
var link = document.createElement('link');
blob = dataURItoBlob($scope.myCroppedImage)
console.log(blob)
};
angular.element(document.querySelector('#fileInput')).on('change',handleFileSelect);
function dataURItoBlob(dataURI) {
// convert base64/URLEncoded data component to raw binary data held in a string
var binary = atob(dataURI.split(',')[1]);
var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
var array = [];
for(var i = 0; i < binary.length; i++) {
array.push(binary.charCodeAt(i));
}
return new Blob([new Uint8Array(array)], {type: mimeString});
}
$scope.upload = function(file) {
//var file = new File(file, "filename");
// Configure The S3 Object
console.log($scope.creds)
AWS.config.update({ accessKeyId: $.trim($scope.creds.access_key), secretAccessKey: $.trim($scope.creds.secret_key) });
AWS.config.region = 'us-east-1';
var bucket = new AWS.S3({ params: { Bucket: $.trim($scope.creds.bucket) } });
if(file) {
//file.name = 'abc';
var uniqueFileName = $scope.uniqueString() + '-' + file.name;
var params = { Key: file.name , ContentType: file.type, Body: file, ServerSideEncryption: 'AES256' };
bucket.putObject(params, function(err, data) {
if(err) {
// There Was An Error With Your S3 Config
alert(err.message);
return false;
}
else {
// Success!
alert('Upload Done');
}
})
.on('httpUploadProgress',function(progress) {
// Log Progress Information
console.log(Math.round(progress.loaded / progress.total * 100) + '% done');
});
}
else {
// No File Selected
alert('No File Selected');
}
}
$scope.uniqueString = function() {
var text = "";
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for( var i=0; i < 8; i++ ) {
text += possible.charAt(Math.floor(Math.random() * possible.length));
}
return text;
}
//for uploading
$scope.handleSave = function(){
$scope.upload(blob);
}
Now, I want to upload this blob on S3 using this, but I am not able to figure out how to upload this blob file to s3 (as I am not getting 'name' in the blob file)
Any help would be really appreciated. Thanks
You can always create file from blob. You can pass file name also.
var file = new File([blob], "filename");
This same file object you can use to upload on s3.
Change your handleSave method to following. File name will be abc.png for now
//for uploading
$scope.handleSave = function(){
blob = dataURItoBlob($scope.myCroppedImage)
$scope.upload(new File([blob], "abc.png"));
}
It is not advisable that you do to put the key
secretAccessKey: $.trim($scope.creds.secret_key)
on the client side ... That is not done !, anyone can manipulate your bucket at will.

Resources