My code is working fine when I zip 3 files around 300kb each and send it to client. Used following links for help:
Dynamically create and stream zip to client
how to convert multiple files to compressed zip file using node js
But as soon as I try to zip 4th file I get "download - Failed Network error" in chrome.
Following is my code:
var express = require('express');
var app = express();
var fileSystem = require('fs');
var Archiver = require('archiver');
var util = require('util');
var AdmZip = require('adm-zip');
var config = require('./config');
var log_file = fileSystem.createWriteStream(__dirname + '/debug.log', {flags : 'a'});
logError = function(d) { //
log_file.write('[' + new Date().toUTCString() + '] ' + util.format(d) + '\n');
};
app.get('/zip', function(req, res, next) {
try {
res = setHeaderOfRes(res);
sendZip(req, res);
}catch (err) {
logError(err.message);
next(err); // This will call the error middleware for 500 error
}
});
var setHeaderOfRes = function (res){
res.setHeader("Access-Control-Allow-Origin", "*"); //Remove this when this is on production
res.setHeader("Content-Type", "application/zip");
res.setHeader("Content-disposition", "attachment;");
return res;
};
var sendZip = function (req, res) {
var filesNotFound = [];
zip.pipe(res);
if (req.query.leapIds) {
var leapIdsArray = req.query.leapIds.split(',');
var i, lengthi;
for (i = 0, lengthi = leapIdsArray.length; i < lengthi; i++) {
try {
var t = config.web.sharedFilePath + leapIdsArray[i] + '.vsdx';
if (fileSystem.statSync(t).isFile()) {
zip.append(new fileSystem.createReadStream(t), {
name: leapIdsArray[i] + '.vsdx'
});
};
} catch (err) {
filesNotFound.push(leapIdsArray[i] + '.vsdx');
}
}
var k, lengthk;
var str = '';
for (k = 0, lengthk = filesNotFound.length; k < lengthk; k++) {
str += filesNotFound[k] +',';
}
if(filesNotFound.length > 0){
zip.append('These file/files does not exist on server - ' + str , { name: 'logFile.log' });
}
zip.finalize();
}
};
I tried zip.file instead of zip.append that didn't work.
I want to zip minimum 10 files of 300kb each and send it to the client. Can anyone please let me know the approach.
Thanks
/********************* Update ****************************************
I was only looking at server.js created in node. Actually the data is sent correctly to client. Angularjs client code seems to be not working for large files.
$http.get(env.nodeJsServerUrl + "zip?leapIds=" + nodeDetails, { responseType: "arraybuffer" }
).then(function (response) {
nodesDetails = response.data;
var base64String = _arrayBufferToBase64(nodesDetails);
function _arrayBufferToBase64(buffer) {
var binary = '';
var bytes = new Uint8Array(buffer);
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary);
}
var anchor = angular.element('<a/>');
anchor.attr({
href: 'data:application/zip;base64,' + base64String,
target: '_blank',
download: $scope.main.routeParams.sectorId + "-ProcessFiles.zip"
})[0].click();
});
This part href: 'data:application/zip;base64,' + base64String, seems to be failing for large data received from server. For small files it is working. Large files it is failing.
Found out.
The problem was not in nodejs zipping logic. That worked perfect.
Issue was in the way I was handling the received response data.
If the data that is received is too large then following code fails
anchor.attr({
href: 'data:application/zip;base64,' + base64String,
target: '_blank',
download: $scope.main.routeParams.sectorId + "-ProcessFiles.zip"
})[0].click();
so the work around is to use blob:
function b64toBlob(b64Data, contentType, sliceSize) {
contentType = contentType || '';
sliceSize = sliceSize || 512;
var byteCharacters = atob(b64Data);
var byteArrays = [];
for (var offset = 0; offset < byteCharacters.length; offset += sliceSize) {
var slice = byteCharacters.slice(offset, offset + sliceSize);
var byteNumbers = new Array(slice.length);
for (var i = 0; i < slice.length; i++) {
byteNumbers[i] = slice.charCodeAt(i);
}
var byteArray = new Uint8Array(byteNumbers);
byteArrays.push(byteArray);
}
var blob = new Blob(byteArrays, { type: contentType });
return blob;
}
var contentType = 'application/zip'
var blob = b64toBlob(base64String, contentType);
saveAs(blob, "hello world.zip");
This link helped me out: How to save binary data of zip file in Javascript?
already answered here: https://stackoverflow.com/a/62639710/8612027
Sending a zip file as binary data with expressjs and node-zip:
app.get("/multipleinzip", (req, res) => {
var zip = new require('node-zip')();
var csv1 = "a,b,c,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
zip.file('test1.file', csv1);
var csv2 = "z,w,x,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
zip.file('test2.file', csv2);
var csv3 = "q,w,e,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
zip.file('test3.file', csv3);
var csv4 = "t,y,u,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
zip.file('test4.file', csv4);
var data = zip.generate({base64:false,compression:'DEFLATE'});
console.log(data); // ugly data
res.type("zip")
res.send(new Buffer(data, 'binary'));
})
Creating a download link for the zip file. Fetch data and convert the response to an arraybuffer with ->
//get the response from fetch as arrayBuffer...
var data = response.arrayBuffer();
const blob = new Blob([data]);
const fileName = `${filename}.${extension}`;
if (navigator.msSaveBlob) {
// IE 10+
navigator.msSaveBlob(blob, fileName);
} else {
const link = document.createElement('a');
// Browsers that support HTML5 download attribute
if (link.download !== undefined) {
const url = URL.createObjectURL(blob);
link.setAttribute('href', url);
link.setAttribute('download', fileName);
link.style.visibility = 'hidden';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}
}
my code in here
here $scope.student_photo is my variable to get encoded string
first i get it in ajax and stored in local db
$http.post(mData.url+'getStudentDetails/',{student_id : student_id} ).success(function(data){
//$scope.student_pic = data.student_details[0].student_pic;
//$scope.student_photo = data.student_details[0].student_photo;
$scope.myImage=data.student_details[0].student_photo;
//alert($scope.myImage);
// var image_stu = data.student_details[0].student_photo;
// localStorage.setItem("imageData", image_stu);
// document.getElementById("img_stu").src='data:image/png;base64,' + image_stu;
//alert("DONE");
var events = data.student_details
var l = events.length;
//alert(l);
db.transaction(function(tx) {
tx.executeSql('SELECT * FROM dashboard ', [], function(tx, results){
if (results.rows.length == 0)
{
tx.executeSql("INSERT INTO dashboard(dashboard_id, stu_name,clas,sec,student_image) VALUES(?,?,?,?,?)", [ student_id,data.student_details[0].student_name, data.student_details[0].class_name, data.student_details[0].section_name,data.student_details[0].student_photo], function(tx, res) {
$scope.dashboardDetails();
//alert('INSERTED');
}, function(e) {
//alert('ERROR: ' + e.message);
});
}
});
});
});
$scope.showScreen = function(screen) {
$state.go('app.'+screen);
}
$scope.printDashboard = function()
{
//alert('ss');
db.transaction(function (tx) {
// tx.executeSql('SELECT * FROM dashboard', [], function (tx, dtresult) {
//alert("dtresult.rows.length" + dtresult.rows.length);
// console.log("dtresult.rows.length" + dtresult.rows.length);
// $scope.totalrecords = dtresult.rows.length;
// });
tx.executeSql('SELECT * FROM dashboard', [], function (tx, dresult) {
console.log("dresult.rows.length" + dresult.rows.length);
dataset = dresult.rows;
console.log(dresult.rows);
for (var i = 0, item = null; i < dresult.rows.length; i++) {
item = dataset.item(i);
$scope.dashboarditems.push({stu_name: item['stu_name'],stu_class: item['clas'],stu_sec: item['Sec'],stu_img: item['student_image']});
//$scope.items.push({id: item['notice_title'], notice:item['notice'], event_date: item['event_date']});
console.log($scope.dashboarditems[0]);
}
$state.go('app.dashboard');
});
});
}
$scope.dashboardDetails = function()
{
var citems = [];
syncWithServer(function(syncSuccess){
$scope.printDashboard();
}, false);
}
Question:
How to display my base64 string to image formate.
If your base64 string is present inside the $scope.student_photo
try the following img tag to display the image in view
<img ng-src="{{'data:image/png;base64,'+student_photo}}">
I always manage pagination with angular
retrieve all the data from the server
and cache it client side (simply put it in a service)
now I have to cope with quite lot of data
ie 10000/100000.
I'm wondering if can get into trouble
using the same method.
Imo passing parameter to server like
page search it's very annoying for a good
user experience.
UPDATE (for the point in the comment)
So a possible way to go
could be get from the server
like 1000 items at once if the user go too close
to the offset (ie it's on the 800 items)
retrieve the next 1000 items from the server
merge cache and so on
it's quite strange not even ng-grid manage pagination
sending parameters to the server
UPDATE
I ended up like:
(function(window, angular, undefined) {
'use strict';
angular.module('my.modal.stream',[])
.provider('Stream', function() {
var apiBaseUrl = null;
this.setBaseUrl = function(url) {
apiBaseUrl = url;
};
this.$get = function($http,$q) {
return {
get: function(id) {
if(apiBaseUrl===null){
throw new Error('You should set a base api url');
}
if(typeof id !== 'number'){
throw new Error('Only integer is allowed');
}
if(id < 1){
throw new Error('Only integer greater than 1 is allowed');
}
var url = apiBaseUrl + '/' + id;
var deferred = $q.defer();
$http.get(url)
.success(function (response) {
deferred.resolve(response);
})
.error(function(data, status, headers, config) {
deferred.reject([]);
});
return deferred.promise;
}
};
};
});
})(window, angular);
(function(window, angular, undefined) {
'use strict';
angular.module('my.mod.pagination',['my.mod.stream'])
.factory('Paginator', function(Stream) {
return function(pageSize) {
var cache =[];
var staticCache =[];
var hasNext = false;
var currentOffset= 0;
var numOfItemsXpage = pageSize;
var numOfItems = 0;
var totPages = 0;
var currentPage = 1;
var end = 0;
var start = 0;
var chunk = 0;
var currentChunk = 1;
var offSetLimit = 0;
var load = function() {
Stream.get(currentChunk).then(function(response){
staticCache = _.union(staticCache,response.data);
cache = _.union(cache,response.data);
chunk = response.chunk;
loadFromCache();
});
};
var loadFromCache= function() {
numOfItems = cache.length;
offSetLimit = (currentPage*numOfItemsXpage)+numOfItemsXpage;
if(offSetLimit > numOfItems){
currentChunk++;
load();
}
hasNext = numOfItems > numOfItemsXpage;
totPages = Math.ceil(numOfItems/numOfItemsXpage);
paginator.items = cache.slice(currentOffset, numOfItemsXpage*currentPage);
start = totPages + 1;
end = totPages+1;
hasNext = numOfItems > (currentPage * numOfItemsXpage);
};
var paginator = {
items : [],
notFilterLabel: '',
hasNext: function() {
return hasNext;
},
hasPrevious: function() {
return currentOffset !== 0;
},
hasFirst: function() {
return currentPage !== 1;
},
hasLast: function() {
return totPages > 2 && currentPage!==totPages;
},
next: function() {
if (this.hasNext()) {
currentPage++;
currentOffset += numOfItemsXpage;
loadFromCache();
}
},
previous: function() {
if(this.hasPrevious()) {
currentPage--;
currentOffset -= numOfItemsXpage;
loadFromCache();
}
},
toPageId:function(num){
currentPage=num;
currentOffset= (num-1) * numOfItemsXpage;
loadFromCache();
},
first:function(){
this.toPageId(1);
},
last:function(){
this.toPageId(totPages);
},
getNumOfItems : function(){
return numOfItems;
},
getCurrentPage: function() {
return currentPage;
},
getEnd: function() {
return end;
},
getStart: function() {
return start;
},
getTotPages: function() {
return totPages;
},
getNumOfItemsXpage:function(){
return numOfItemsXpage;
},
search:function(str){
if(str===this.notFilterLabel){
if(angular.equals(staticCache, cache)){
return;
}
cache = staticCache;
}
else{
cache = staticCache;
cache = _.filter(cache, function(item){ return item.type == str; });
}
currentPage = 1;
currentOffset= 0;
loadFromCache();
}
};
load();
return paginator;
}
});
})(window, angular);
server side with laravel (All the items are cached)
public function tag($page)
{
$service = new ApiTagService(new ApiTagModel());
$items = $service->all();
$numOfItems = count($items);
if($numOfItems > 0){
$length = self::CHUNK;
if($length > $numOfItems){
$length = $numOfItems;
}
$numOfPages = ceil($numOfItems/$length);
if($page > $numOfPages){
$page = $numOfPages;
}
$offSet = ($page - 1) * $length;
$chunk = array_slice($items, $offSet, $length);
return Response::json(array(
'status'=>200,
'pages'=>$numOfPages,
'chunk'=>$length,
'data'=> $chunk
),200);
}
return Response::json(array(
'status'=>200,
'data'=> array()
),200);
}
The only trouble by now is managing filter
I've no idea how to treat filtering :(