I've got an app that connects to a SQL Server database. It has 3 post API's that are going to get A LOT of traffic. I'm having trouble with connection timeout and want to make sure I have Connection Pooling setup correctly or figure out where I'm going wrong. I have a DBConfig.js file setup as follows
var sql = require('mssql');
let curacaopool = null;
const curacao = () => {
let config;
if (null !== curacaopool) {
return curacaopool;
}
if (process.env.APP_ENV === 'prod') {
config = {
user: 'user',
password: 'password',
server: 'IP Address',
port: 51678,
database: 'Database',
connectionTimeout: 15000,
//requestTimeout: 45000,
pool: {
max: 10,
min: 1,
idleTimeoutMillis: 30000
}
};
}
curacaopool = new sql.ConnectionPool(config).connect();
return curacaopool;
};
module.exports = { curacao };
My API's are setup as follows. I have 3 of them. The first API gets called and returns an ID then the next two API's get called
APIs
var express = require('express');
var router = express.Router();
var sql = require('mssql');
const { curacao } = require('../../../repository/getDbConfig');
router.post('/workstation', async function (req, res, next) {
curacao()
.then((pool) => pool.request())
.then((request) => {
for (const [key, value] of Object.entries(dbColumns)) {
if (typeof value !== 'undefined' && value !== '') {
request.input(key, dbTypes[key], value);
}
}
request.input('ip_address', sql.VarChar, ip_address);
request.output('workstationId', sql.Int);
request.execute('Gumshoe.dbo.InsertWorkstations', (err, result) => {
if (err) {
res.status(500).json({ 'Workstation Error': err });
}
res.status(200).json({
status: 200,
message: 'Data Saved',
workstationId: result.output.workstationId
});
});
})
.catch((err) => {
console.log('Workstations Error', err);
res.status(500).json({ 'Workstations Error': err });
});
});
router.post('/packages', async function (req, res, next) {
const sentdata = req.body;
var table = new sql.Table('Database.dbo.Packages');
table.create = false;
table.columns.add('workstation_id', sql.Int, { nullable: true });
table.columns.add('pkg_name', sql.VarChar, { nullable: true });
table.columns.add('pkg_ver', sql.VarChar, { nullable: true });
table.columns.add('appWizName', sql.VarChar, { nullable: true });
table.columns.add('appWizPublisher', sql.VarChar, { nullable: true });
table.columns.add('appWizInstalledOn', sql.Date, { nullable: true });
table.columns.add('appWizVersion', sql.VarChar, { nullable: true });
table.columns.add('appWizSize', sql.VarChar, { nullable: true });
table.columns.add('customTrack1', sql.VarChar, { nullable: true });
table.columns.add('customTrack2', sql.VarChar, { nullable: true });
table.columns.add('customTrack3', sql.VarChar, { nullable: true });
if (sentdata.length > 0) {
sentdata.forEach((item) => {
if (parseInt(item.workstation_id) > 0 && item.pkg_name.length > 1) {
return table.rows.add(
item.workstation_id,
item.pkg_name,
item?.pkg_ver || null,
item?.appWizName || null,
item?.appWizPublisher || null,
item?.appWizInstalledOn ? (dateCheck(item.appWizInstalledOn) === true ? item.appWizInstalledOn : null) : null,
item?.appWizVersion || null,
item?.appWizSize || null,
item?.customTrack1 || null,
item?.customTrack2 || null,
item?.customTrack3 || null
);
}
});
curacao()
.then((pool) => pool.request())
.then((request) =>
request
.bulk(table, (err, result) => {
if (err) {
console.log('Package Bulk Error:', err);
res.status(500).json({ 'Bulk Packages': err });
}
res.status(200).json(result);
})
.catch((err) => {
console.log('Packages Error', err);
res.status(500).json({ 'Packages Error': err });
})
);
} else {
console.log('Package sent data was empty Array', sentdata);
res.status(500).send('Package empty Array');
}
});
The first API calls a stored proc, this stored proc deletes data from the tables for the next two APIs. The next two (both setup the same) do Bulk Inserts to two other tables
I'm having a lot of trouble with Timeout Errors
Webserver
returned:{"status":500,"pool_error":{"code":"ETIMEOUT","originalError":{"message":"Timeout:
Request failed to complete in 15000ms
I've tried increasing the pool Max to 25. I've tried increased the requestTimeout value. Both of those haven't worked or helped. These work in test when not under heavy load so wondering what I look to next to get this resolved.
Thanks for any assistance
Related
this is the route:
router.post(
'/',
[ auth,
check('status', 'Status is required').not().isEmpty(),
check('skills', 'Skills is required').not().isEmpty(),
],
async (req, res) => {
const errors = validationResult(req);
if (!errors.isEmpty()) {
return res.status(400).json({ errors: errors.array() });
}
// destructure the request
const {
website,
skills,
youtube,
twitter,
instagram,
linkedin,
facebook,
// spread the rest of the fields we don't need to check
...rest
} = req.body;
// build a profile
const profileFields = {
user: req.user.id,
website:
website && website !== ''
? normalize(website, { forceHttps: true })
: '',
skills: Array.isArray(skills)
? skills
: skills.split(',').map((skill) => ' ' + skill.trim()),
...rest
};
// Build socialFields object
const socialFields = { youtube, twitter, instagram, linkedin, facebook };
// normalize social fields to ensure valid url
for (const [key, value] of Object.entries(socialFields)) {
if (value && value.length > 0)
socialFields[key] = normalize(value, { forceHttps: true });
}
// add to profileFields
profileFields.social = socialFields;
try {
// Using upsert option (creates new doc if no match is found):
let profile = await Profile.findOneAndUpdate(
{ user: req.user.id },
{ $set: profileFields },
{ new: true, upsert: true, setDefaultsOnInsert: true }
);
return res.json(profile);
} catch (err) {
console.error(err.message);
return res.status(500).send('Server Error');
}
}
);
this is Axios:
axios.post('/api/profile',profileData, {headers:{
'x-auth-token': localStorage.getItem('jwtToken')
}}).then(data => console.log(data)).catch(e => console.error(e))
this is problem:
pending
Have you tried to use debugger in order to follow software flow?
Have you any log from express side?
I'd start checking if let profile = await Profile.findOneAndUpdate is reached in order to define if the problem starts on db request.
At this stage I would say that the execution is pending waiting for db timeout
I am stuck on this problem for 2 days. I am sending POSTrequest from frontend to the backend (and other GET requests too but the problem is only with POST). However, when my data goes to the backend it does not post anything to the rest api even though response is 200 OK. That's why when in response it should have given the posted data, it can't find it and gives null. This is my POST code in backend index.js:
const { response, request } = require('express');
require('dotenv').config()
const express = require('express');
const morgan = require('morgan');
const Contact = require('./models/contact.cjs');
const cors = require('cors')
const app = express();
app.use(express.json())
app.use(express.static('build'))
app.use(cors())
morgan.token('body', req => {
return JSON.stringify(req.body)
})
app.use(morgan(':method :url :status :res[content-length] - :response-time ms :body'));
const generateId = () => {
const randNum = Math.floor(Math.random() * 5000)
return randNum;
}
app.post('/api/persons', (req, res) => {
const body = req.body
console.log(body)
if (!body.name || !body.number) {
return res.status(400).json({
error: "missing data"
})
} else if (Contact.find({name: body.name})) {
Contact.findOneAndUpdate({name: body.name}, {$set: {number: body.number}}, {new:true})
.then(updatedContacts =>
res.json(updatedContacts)
)
.catch(err => console.log(err))
} else {
const contact = Contact({
id: generateId(),
name: body.name,
number: body.number,
date: new Date()
})
contact.save()
.then(savedContact => {
console.log(savedContact)
res.json(savedContact)
})
.catch(err => {
console.log(err)
})
}
})
const PORT = process.env.PORT
app.listen(PORT, () => {
console.log(`Server is working on ${PORT}`)
})
and this is how my frontend sends data to backend: contacts.js:
const create = (newObject) => {
const readyToPost = {
method: 'post',
url: `${baseUrl}`,
data: newObject,
headers: {'Content-Type': 'application/json'},
json: true
}
const request = axios(readyToPost)
return request.then(response => {
console.log(response.data)
return response.data
})
.catch(err => {
console.log(err)
})
}
And this is my react app's frontend.
Any ideas about why my data becomes null?
Any help would be appreciated!
Due to the synchronous nature of your code, the condition Contact.find({name: body.name}) was always returning the Query object which is true due to which the else if block was getting executed even when there was no such document. After entering the else if block, since there was no match, so findOneAndUpdate() was returning null.
Use findOne() instead of find(). find() returns a cursor which is empty but true whereas findOne() returns the first document matched (if matched) or else it will return null (if not matched).
// index.js (Backend)
app.post("/api/persons", async (req, res) => {
const body = req.body;
if (!body.name || !body.number) {
return res.status(400).json({
error: "missing data",
});
}
// Using findOne() instead of find(). Returns null if record not found.
const existing = await Contact.findOne({ name: body.name });
if (existing) {
Contact.findOneAndUpdate(
{ name: body.name },
{ $set: { number: body.number } },
{ new: true }
)
.then((updatedContacts) => {
console.log(updatedContacts);
res.status(200).json(updatedContacts);
})
.catch((err) => console.log(err));
} else {
const contact = Contact({
id: generateId(),
name: body.name,
number: body.number,
date: new Date(),
});
contact
.save()
.then((savedContact) => {
console.log(savedContact);
res.status(201).json(savedContact);
})
.catch((err) => {
console.log(err);
});
}
});
I am trying to get socket.io to work with Heroku but it doesn't play well.
the problem: the code below work flawlessly in dev but after uploaded to Heroku it's working sometimes,
1.don't see anything weird on logs.
2.the data saved to DB and will appear after refresh
3.refresh helps to sockets sometimes.
4.there is no pattern to it, sometime it will work ok for an hour and some time won't last a minute
5.heroku features:enable http-session-affinity also done
server:
const mongoose = require("mongoose");
const Rooms = mongoose.model("Rooms");
const Chat = mongoose.model("Chats");
const jwt = require("jwt-then");
const socketChat = (app, io) => {
io.use(async (socket, next) => {
try {
const token = socket.handshake.query.token;
const payload = await jwt.verify(token, process.env.SECRET_KEY);
socket.userId = payload.id;
socket.id = payload.id;
socket.name = payload.username;
console.log({ socketisd: socket.userId, name: socket.name });
next();
} catch (err) { }
});
io.on("connection", (socket) => {
console.log("Connected: " + socket.name);
socket.on("disconnect", () => {
console.log("Disconnected: " + socket.name);
});
socket.on("joinRoom", async ({ roomId },callback) => {
socket.join(roomId);
console.log(` ${socket.name} joined room: ` + roomId);
socket.to(roomId).emit("live", { name: socket.name, live: true, roomId });
callback({
status: "ok"
});
});
socket.on("leaveRoom", async ({ roomId },callback) => {
socket.leave(roomId);
console.log(` ${socket.name} left room: ` + roomId);
socket.to(roomId).emit("live", { name: socket.name, live: false, roomId });
callback({
status: "ok"
});
});
socket.on("typing", async ({ msg, roomId }) => {
let name = "";
if (msg.text && msg.text.trim().length > 0) {
let length = msg.text.length;
name = length > 0 ? socket.name : "";
}
socket.to(roomId).emit("typingclient", { name });
});
socket.on(
"chatroomMessage",
async ({ roomId, message, name, profileImg, timestamp, type, date }) => {
if (message.trim().length > 0) {
io.to(roomId).emit("newMessage", {
roomId,
user: socket.userId,
message,
name,
type,
date,
profileImg,
timestamp,
recived: true,
});
let room = await Rooms.findById(roomId).populate("messages");
if (type === "reject") {
await Chat.findOneAndUpdate(
{ roomId, type: "dateConfirm" },
{ type: "reject", message },
{ new: true }
);
}
else {
const newMessage = new Chat({
roomId,
date,
type,
user: socket.userId,
message,
name,
profileImg,
timestamp,
recived: true,
});
await newMessage.save();
room.messages.push(newMessage);
await room.save();
}
let theOtherGuy =await room.users.find((user) => user != socket.userId);
io.to(theOtherGuy).emit("room", room);
}
}
);
});
};
module.exports = socketChat;
client:
/**
* Sends message with emit socket to server
* #param {Object} event Default Browser Event Object
* #param {String} text content of message
* #param {String} date Date for schedualing
* #param {String} type type of the message (reject,request etc...)
*/
const sendMessage = (event, text, date = null, type = null) => {
event && event.preventDefault();
if (socket) {
socket.emit("chatroomMessage", {
roomId,
date,
type,
name: currentUser.user.username,
profileImg: currentUser.user.profileImageUrl,
timestamp: new Date(),
recived: false,
message: text,
});
setText("");
socket.emit("typing", {
msg: "",
roomId,
});
}
};
React.useEffect(() => {
if (socket) {
socket.emit("joinRoom", {roomId},(answer)=>
console.log("joinRoom",roomId,answer)
);
socket.on("newMessage", (message) => {
console.log({message})
if (message.type === "reject")
setMessages((prevMessages) => [...prevMessages.filter(m => m.type !== 'dateConfirm'), message]);
else
setMessages((prevMessages) => [...prevMessages, message]);
});
socket.on("live", (message) => {
console.log(message)
message.live ? setSucess(`user ${message.name} has connected`) : setErr(`user ${message.name} has left`)
});
socket.on("typingclient", (name) => {
setTyping(name);
});
}
return () => {
if (socket) {
socket.emit("leaveRoom", {roomId},(answer)=>
console.log("leaveRoom",roomId,answer)
);
}
//Component Unmount
};
//eslint-disable-next-line
}, [socket]);
and main where i define my socket:
const [socket, setSocket] = React.useState(null);
const setupSocket = () => {
console.log("socket4")
const token = sessionStorage.getItem("jwtToken");
if (token && !socket) {
const newSocket = io("/", {
query: {
token: sessionStorage.getItem("jwtToken"),
},
path: '/socket'
});
newSocket.on("disconnect", () => {
// setSocket(null);
// makeToast("error", "Socket Disconnected!");
});
newSocket.on("connect", () => {
// makeToast("success", "Socket Connected!");
console.log("Socket Connected");
});
setSocket(newSocket);
}
};
React.useEffect(() => {
if (currentUser && !socket) setupSocket();
//eslint-disable-next-line
}, [currentUser, socket]);
ststic.json:
{
"root":"build/",
"routes":{
"/**":"index.html"
},
"proxies":{
"/api/":{"origin":"${API_URL}"},
"/socket/":{"origin":"${SOCKET_URL}"}
}
}
it looks like the Io object did not like
socket.id = payload.id;
I have removed it and everything is working now.
I think it may have resulted from different keys in the Io object resulting in unexpected behavior.
I'm receiving an error when trying to associate a saved game to the user that saves it. The error says "cannot read property push of undefined"
The user, and game can be read in the console. I think it may have something to do with the user model during the initial creation of the user, however I can't be sure. I did notice if I try to console.log(user.favGames) it will be returned undefined.
I've tried everything I can think of, I've re-written the controller roughly 10 times, to no avail.
user model
const mongoose = require('mongoose')
const bcrypt = require('bcrypt')
const SALT_ROUNDS = 6
const Schema = mongoose.Schema
const userSchema = new Schema(
{
username: { type: String, unique: true },
email: { type: String, unique: true, unique: true },
password: { type: String, required: true },
avatar: { type: String },
favGames: { type: Schema.Types.ObjectId, ref: 'Game', default: null },
comments: { type: Schema.Types.ObjectId, ref: 'Comment', default: null }
},
{
timestamps: true
}
)
userSchema.set('toJSON', {
transform: function(doc, ret) {
delete ret.password
return ret
}
})
userSchema.pre('save', function(next) {
const user = this
if (!user.isModified('password')) return next()
bcrypt.hash(user.password, SALT_ROUNDS, function(err, hash) {
if (err) return next()
user.password = hash
next()
})
})
userSchema.methods.comparePassword = function(tryPassword, cb) {
bcrypt.compare(tryPassword, this.password, cb)
}
module.exports = mongoose.model('User', userSchema)
game model
const mongoose = require('mongoose')
const Schema = mongoose.Schema
let gameSchema = new Schema({
name: { type: String, required: true },
boxArtUrl: { type: String, required: true },
twitchID: { type: String, required: true },
comments: { type: Schema.Types.ObjectId, ref: "Comment"}
})
module.exports = mongoose.model('Game', gameSchema)
game router
const express = require('express')
const router = express.Router()
const gamesCtrl = require('../../controllers/gameCtrl')
function isAuthed(req, res, next) {
if (req.user) return next()
return res.status(401).json({ msg: 'Unauthorized ' })
}
router.get('/')
router.post('/', isAuthed, gamesCtrl.addGame)
module.exports = router
game controller
const User = require('../models/user')
const Game = require('../models/Game')
function addGame(req, res) {
Game.create({
name: req.body.name,
twitchID: req.body.id,
boxArtUrl: req.body.box_art_url
})
.then(game => {
User.findById(req.user._id)
.then(user => {
console.log(game)
console.log(user.favGames)
// user.favGames.push(game)
// user.save()
})
.catch(err =>
console.log('error when updating user with new game', err)
)
})
.catch(err => console.log('error saving game', err))
}
module.exports = {
addGame
}
the error is flagged in my controller at user.favGames.push(game). Note that when a user creates a profile there are no games associated with their profile. I'm pretty sure I'm calling on the actual data instance of the model, not the model itself. Thanks in advance for your assistance.
Your favGames (and also comments) must be defined as array in user model like this.
const userSchema = new Schema(
{
username: { type: String, unique: true },
email: { type: String, unique: true, unique: true },
password: { type: String, required: true },
avatar: { type: String },
favGames: [{ type: Schema.Types.ObjectId, ref: 'Game', default: null }],
comments: [{ type: Schema.Types.ObjectId, ref: 'Comment', default: null }]
},
{
timestamps: true
}
)
Also user.save() returns a promise, so you need use then block, or await.
So the addGame function must be like this (I converted the code to async/await)
async function addGame(req, res) {
try {
let game = await Game.create({
name: req.body.name,
twitchID: req.body.id,
boxArtUrl: req.body.box_art_url
});
let user = await User.findById(req.user._id);
if (user) {
user.favGames.push(game);
await user.save();
res.status(200).send("game and user saved");
} else {
console.log("user not found");
res.status(404).send("user not found");
}
} catch (err) {
console.log("Err: ", err);
res.status(500).send("Something went wrong");
}
}
Looks like it's a matter of checking to see if it exists:
User.findById(req.user._id)
.then(user => {
if (!Array.isArray(user.favGames)) {
user.favGames = [];
}
user.favGames.push(game);
user.save();
})
I am using this code to connect to my SQL Server and retrieve some data which works fine, if I only call the code once. If I call it twice I get this error:
ConnectionError: Already connecting to database! Call close before connecting to different database.at ConnectionPool._connect
But I am closing the conn after the call so I'm not sure what I am missing.
var sql = require('mssql');
const pool = new sql.ConnectionPool({
user: 'sa',
password: 'password',
server: '192.168.1.2',
database: 'demo',
options: {
encrypt: false
}
})
var conn = pool;
module.exports.getCounter = function( query){
conn.connect().then(function (err) {
var req = new sql.Request(conn);
req.query(query).then(function (result) {
console.log(result.recordset);
return result.recordset;
conn.close();
})
.catch(function (err) {
console.log(err);
conn.close();
});
})
.catch(function (err) {
console.log(err);
})};
You're returning the value before closing the connection, hence the function terminates before reaching that line. So just move the return statement below your conn.close(). The other issues you might have afterwards is that you might be calling your function twice before one executes and terminates completely, since those calls are asynchronous.
You might have to set your getCounter function as a Promise, so that you can wait for its completion/failure before calling it again. Off the top of my head in your example:
const getCounter = () => new Promise((resolve,reject) => {
conn.connect().then(function (err) {
var req = new sql.Request(conn);
req.query(query).then(function (result) {
conn.close();
resolve(result);
})
.catch(function (err) {
conn.close();
reject(err);
});
})
})
You can call your function afterwards as getCounter().then((result) => {...})
Here is another way to solve it which might be helpful for others.
const sql = require('mssql')
let connectionPoolConfig = {
user: 'sa',
password: 'password',
server: '192.168.1.2',
database: 'demo',
options: {
encrypt: false
}
}
let connectionPoolPromise = null
let connectionPoolObj = null
let getOrCreatePool = async () => {
if (connectionPoolObj) {
return connectionPoolObj
} else if (!connectionPoolPromise) {
connectionPoolPromise = new sql.ConnectionPool(connectionPoolConfig).connect()
}
connectionPoolObj = await connectionPoolPromise
return connectionPoolObj
}
let query = async(sql) => {
const pool = await getOrCreatePool()
return await pool.request().query(sql)
}
module.exports = {
query: query
}
And here is how to call it
let testCallerSQL = async () => {
try {
const res = await sqlUtil.query('select * from mytable')
console.log(res.recordset)
} catch(err) {
console.log(err)
} finally {
}
}