I am facing slow query execution time issues with MongoDB.
My Basic scenario is, I am developing a Classifieds website on Sails ( Node ) using MongoDB (Linux-x86_64-3.4.4) as Database.
I have two collections
ad - To save advertisement details ( 194419 records / 753.84 mb )
ad_attrs - To save advertisement attributes details ( 689883 records / 148.75 mb )
We have 2 Different category for users - Personal and Professional. So, we save it to ad collection, Each record contains its type as Personal or Professional.
Now, I want to display total counts of ad based on user's search filters like below
So, I have used below queries to make it possible. but, It been a problem with its performance now.
When I try to Query using Lookup with attrs it's taking 7.5 - 12 Seconds and When I only Query Ad table it's taking 0.2-1.2 seconds.
db.ad.aggregate(
[
{
"$match":{
"$and":[
{
"is_active":{
"$eq":1
}
},
{
"is_delete":{
"$eq":0
}
},
{
"is_block":{
"$eq":0
}
},
{
"is_online":{
"$eq":1
}
},
{
"ad_type":{
"$eq":"offers"
}
}
]
}
},
{
"$lookup":{
"from":"ad_attr",
"localField":"_id",
"foreignField":"ad_id",
"as":"attrs"
}
},
{
"$addFields":{
"totalAttrs":{
"$size":"$attrs"
}
}
},
{
"$project":{
"ad_user_type":1,
"attrs":1
}
},
{
"$group":{
"_id":null,
"cnt":{
"$sum":1
},
"pro":{
"$sum":{
"$cond":[
{
"$eq":[
"$ad_user_type",
"professional"
]
},
1,
0
]
}
},
"per":{
"$sum":{
"$cond":[
{
" $eq":[
"$ad_user_type",
"personal"
]
},
1,
0
]
}
}
}
}
]
)
Collection Structure is as below
Ad Collection
{
attributes: {
_id: {
type: 'string'
},
user_id: {
type: 'string'
},
title: {
type: 'string'
},
description: {
type: 'longtext'
},
price: {
type: 'float'
},
price_max: {
type: 'float'
},
ad_type: {
type: 'string'
},
ad_user_type: {
type: 'string'
},
zipcode: {
type: 'string'
},
lat: {
type: 'float'
},
long: {
type: 'float'
},
loc: {
type: 'object'
},
cityzip: {
type: 'string'
},
town: {
type: 'string'
},
town_slug: {
type: 'string'
},
region: {
type: 'string'
},
region_slug: {
type: 'string'
},
department: {
type: 'string'
},
department_slug: {
type: 'string'
},
address: {
type: 'longtext'
},
telephone: {
type: 'string'
},
is_account_address: {
type: 'integer',
defaultsTo: 1
},
is_phone_hide: {
type: 'integer',
defaultsTo: 0
},
video: {
type: 'string'
},
is_video_check: {
type: 'integer',
defaultsTo: 0
},
is_urgent: {
type: 'integer',
defaultsTo: 0
},
additional_picture: {
type: 'integer',
defaultsTo: 0
},
is_in_showcase: {
type: 'integer',
defaultsTo: 0
},
showcase_date: {
type: 'date'
},
showcase_duration: {
type: 'string',
defaultsTo: ""
},
showcase_duration_id: {
type: 'string'
},
is_in_top_list: {
type: 'integer',
defaultsTo: 0
},
top_list_date: {
type: 'date',
defaultsTo: ''
},
top_list_duration: {
type: 'string',
defaultsTo: ""
},
top_list_duration_id: {
type: 'string'
},
is_direct_sale: {
type: 'integer',
defaultsTo: 0
},
is_active: {
type: 'integer',
defaultsTo: 0
},
is_block: {
type: 'integer',
defaultsTo: 0
},
post_on_from_backend: {
type: 'date',
defaultsTo: ''
},
post_on: {
type: 'date',
defaultsTo: ''
},
post_end_on: {
type: 'date',
defaultsTo: ''
},
payment_status: {
type: 'string',
defaultsTo: 'pending'
},
price_token: {
type: 'string',
},
total_views: {
type: 'integer',
defaultsTo: 0
},
total_clicks: {
type: 'integer',
defaultsTo: 0
},
is_online: {
type: 'integer',
defaultsTo: 0
},
is_change_req: {
type: 'integer',
defaultsTo: 0
},
have_shop: {
type: 'integer',
defaultsTo: 0
},
shop_id: {
type: 'string',
defaultsTo: 0
},
slug: {
type: 'string'
},
category: {
type: 'string'
},
category_price: {
type: 'integer',
defaultsTo: 0
},
parent_category: {
type: 'string'
},
user_ip: {
type: 'string'
},
is_delete: {
type: 'integer',
defaultsTo: 0
},
product_name: {
type: 'string'
},
product_price: {
type: 'float',
defaultsTo: 0
},
original_item_quantity: {
type: 'float',
defaultsTo: 0
},
item_quantity: {
type: 'float',
defaultsTo: 0
},
shipping_type: {
type: 'string'
},
shipping_value_simple: {
type: 'float',
defaultsTo: 0
},
shipping_value_variable_min_1: {
type: 'integer',
defaultsTo: 0
},
shipping_value_variable_max_1: {
type: 'integer',
defaultsTo: 0
},
shipping_value_variable_1: {
type: 'float',
defaultsTo: 0
},
shipping_value_variable_min_2: {
type: 'integer',
defaultsTo: 0
},
shipping_value_variable_max_2: {
type: 'integer',
defaultsTo: 0
},
shipping_value_variable_2: {
type: 'float',
defaultsTo: 0
},
shipping_value_variable_min_3: {
type: 'integer',
defaultsTo: 0
},
shipping_value_variable_max_3: {
type: 'integer',
defaultsTo: 0
},
shipping_value_variable_3: {
type: 'float',
defaultsTo: 0
},
seller_paypal_url: {
type: 'string',
defaultsTo: ""
},
direct_sale_opt_1: {
type: 'string',
defaultsTo: ""
},
direct_sale_value_1: {
type: 'longtext',
defaultsTo: ""
},
direct_sale_opt_2: {
type: 'string',
defaultsTo: ""
},
direct_sale_value_2: {
type: 'longtext',
defaultsTo: ""
},
direct_sale_opt_3: {
type: 'string',
defaultsTo: ""
},
have_update: {
type: 'integer',
defaultsTo: 0
},
update_payment_status: {
type: 'string',
defaultsTo: ""
},
direct_sale_value_3: {
type: 'longtext',
defaultsTo: ""
},
attrs: {
collection: 'ad_attr',
via: 'ad_id'
},
reference: {
type: 'longtext',
defaultsTo: ""
},
emp_id: {
type: 'string',
defaultsTo: "0"
},
},
}
Ad Attribtues Collection
attributes: {
ad_id:{
model:'ad'
},
cat_id:{
type:'string',
},
attr_id:{
type:'string',
},
value:{
type:'string',
},
integer_value:{
type:'float',
defaultsTo: 0,
}
}
Index Added for Collections
Ad - I have added indexes to all attributes those are used for Condition
Id , loc , post_on, price, ad_type, is_delete, is_active, is_online, is_block, ad_user_type, attrs, category, parent_category, region_slug, department_slug, town_slug, slug
Ad Attributes
Id, attr_id, value, ad_id, attrs
Server Configurations - It's fine for the current database size
100 GB SSD
8 GB RAM
4 Core Processor
I have done so far
- Google for all possibilities
- Removing attributes those are not needs to query for count and list
I want you to help me with below questions
Shall I improve my Collection structure to make it faster?
Shall I move some attributes to another collection, which are not used to Query for count?
Shall I need to upgrade Mongo config settings or what?
Shall I need to upgrade my server hardware?
Experts, I need your help to resolve this. Thanks in advance! :)
Related
This is my first post so please bear with me. I am building a LinkedIn clone and I am trying to keep track of the work experience, projects and courses of the users, and those will be kept in an array of objects inside of the User schema. Now let's say a user will try to add or update one of the elements in one of those arrays, I have the user ID and I am passing it to findOneAndUpdate() as the filter.
Here is my User schema:
const userSchema = new mongoose.Schema({
user_id: {
type: String,
required: [true, 'User ID required.'],
unique: true,
immutable: true,
},
name: {
type: String,
required: [true, 'Name required.'],
},
email: {
type: String,
required: [true, 'Email required.'],
unique: true,
lowercase: true,
immutable: true,
},
title: {
type: String,
},
location: {
type: String,
},
phone_number: {
type: String,
},
contact_email: {
type: String,
},
photo: {
type: String,
},
website: {
type: String,
},
backdrop: {
type: String,
},
summary: {
type: String,
},
work: {
type: String,
},
connections: {
type: Number,
},
projects: [
{
title: {
type: String,
},
description: {
type: String,
},
start_date: {
type: Date,
},
end_date: {
type: Date,
},
technologies: {
type: String,
},
picture: {
type: String,
},
},
],
skills: [{
skill: {
name: {
type: String,
},
level: {
type: String,
},
},
}],
experience: [
{
company: {
type: String,
},
logo: {
type: String,
},
title: {
type: String,
},
location: {
type: String,
},
start_date: {
type: Date,
},
end_date: {
type: Date,
},
description: {
type: String,
},
},
],
education: [
{
school: {
type: String,
},
logo: {
type: String,
},
degree: {
type: String,
},
location: {
type: String,
},
start_date: {
type: Date,
},
end_date: {
type: Date,
},
description: {
type: String,
},
},
],
languages: [
{
name: {
type: String,
},
level: {
type: String,
},
},
],
awards: [
{
title: {
type: String,
},
date: {
type: Date,
},
awarder: {
type: String,
},
summary: {
type: String,
},
},
],
courses: [
{
title: {
type: String,
},
number: {
type: String,
},
school: {
type: String,
},
start_date: {
type: Date,
},
end_date: {
type: Date,
},
description: {
type: String,
},
},
],
});
And in my UserController.ts file, I tried using this:
const updateUser = async (req: Request, res: Response) => {
try {
const filter = { user_id: req.body.user_id };
const update = req.body;
const updatedUser = await User.findOneAndUpdate(filter, update, {
new: true,
upsert: true,
});
res.status(201).json({
status: 'success',
data: {
user: updatedUser,
},
});
} catch (err) {
res.status(400).json({
status: `ERROR: ${err}`,
message: 'error updating user',
});
}
};
And in my request using the format of the schema but that didn't work out as expected. I know mongoose will automatically give it an _id field to each of the individual objects in the array, but again, I have had no luck updating them. I tried sending a PATCH request with this as the body to add a skill like so:
{
"user_id": "xxxxxxxxxxxxxxxx",
"title": "Mr.",
"skills" : {
"name": "Flute",
"level": "Novice"
}
}
And this was the response I got. It created a skill but didnt add the data in the skill object:
{
"status": "success",
"data": {
"user": {
"_id": "63d3715f2ef9698667230a53",
"user_id": "xxxxxxxxxxxxxxxx",
"name": "Jonathan Abitbol",
"email": "yoniabitbol1#gmail.com",
"projects": [],
"skills": [
{
"_id": "63d4068d2df30c9e943e4608"
}
],
"experience": [],
"education": [],
"languages": [],
"awards": [],
"courses": [],
"__v": 0,
"title": "Mr."
}
}
}
Any help on how to add/edit the nested objects would be appreciated.
I am following a course on react native and We are using Sanity as our backend. I have already set the schemas and made the adjustments in my Sanity Studio.
HERE IS MY FEATURED SCHEMA CODE:
export default {
name: 'featured',
title: 'featured Menu Categories',
type: 'document',
fields: [
{
name: 'name',
type: 'string',
title: 'Featured category name',
validation: (Role) => Role.required(),
},
{
name: 'short_description',
type: 'string',
title: 'Short description',
validation: (Role) => Role.max(200),
},
{
name: 'restuarants',
type: 'array',
title: 'Restuarants',
of: [{ type: 'reference', to: [{ type: 'restuarant' }] }],
},
],
};
HERE IS MY RESTAURANT SCHEMA CODE:
export default {
name: 'restaurant',
title: 'Restuarant',
type: 'document',
fields: [
{
name: 'name',
type: 'string',
title: 'Restuarant name',
validation: (Role) => Role.required(),
},
{
name: 'short_description',
type: 'string',
title: 'Short description',
validation: (Role) => Role.max(200),
},
{
name: 'image',
type: 'image',
title: 'Image of the Restuarant',
},
{
name: 'lat',
type: 'number',
title: 'latitude of the restaurant',
},
{
name: 'long',
type: 'number',
title: 'longitude of the restaurant,
},
{
name: 'address',
type: 'string',
title: 'Address of the Restuarant',
validation: (Role) => Role.required(),
},
{
name: 'rating',
type: 'number',
title: 'Enter a rating of (1 - 5)',
validation: (Role) =>
Role.required()
.min(1)
.max(5)
.error('please enter a number between 1 - 5'),
},
{
name: 'type',
type: 'string',
title: 'Category',
validation: (Role) => Role.required(),
type: 'reference',
to: [{ type: 'category' }],
},
{
name: 'dishes',
type: 'array',
title: 'Dishes',
of: [{ type: 'reference', to: [{ type: 'dish' }] }],
},
],
};
I also did one for the dish and category.
Then I went to my Sanity Studio to fill in my restaurant schema fields;
After I went to my Vision Board in Sanity Studio and did a query(Just like the instructor):
*[_type == "featured"]{
...,
restuarants[]=> {
...,
dishes[]=> ,
type=> {
name
}
},
}
And I got an error of:
unexpected token ",", expected expression ;
I did what any developer would have done if they got an error. I double-checked my code and compared it to the instructor in the video. (I still got the error). Then I googled it(And found no answer). It's been 2 days now and I still haven't found anything.
This is my first querying in Sanity and I am not sure what I am doing wrong with my query. Can anyone please help me? Thanks.
As the error mentioned, you missed a ' in your schema code.
Keep a ' after restaurant
{
name: 'long',
type: 'number',
title: 'longitude of the restaurant',
},
No worries I found the answer to my problem. The problem was with my querying.
HERE WAS MY CODE BEFORE:
*[_type == "featured"]{
...,
restuarants[]=> {
...,
dishes[]=> ,
type=> {
name
}
},
}
THIS IS HOW I WAS SUPPOSED TO WRITE IT:
*[_type == "featured"]{
...,
restuarants[]-> {
...,
dishes[] -> ,
type-> {
name
}
},
}
I was supposed to use a straight line instead of an equal sign.
I hope this helps anyone who runs into this problem
i want to set type of serverNames as a array of string but in watermelon it supports only boolean, string and number ,anybody help me over this
export const settingSchema = tableSchema({
name: 'SETTINGS',
columns: [
{ name: 'serverId', type: 'number' },
{ name: 'server', type: 'string' },
{ name: 'base_url', type: 'string' },
{ name: 'serverNames', type: 'string', },//<-----serverNames,type:'[string]
{ name: 'amberIcon', type: 'boolean' },
{ name: 'showAllResident', type: 'boolean' },
{ name: 'showAllTask', type: 'boolean' },
{ name: 'filterStart', type: 'number' },
{ name: 'filterEnd', type: 'number' },
{ name: 'searchedText', type: 'string' },
{ name: 'currentTab', type: 'number' },
{ name: 'appInit', type: 'boolean' },
{ name: 'performRefresh', type: 'boolean' },
{ name: 'slowMessageDisplay', type: 'boolean' },
{ name: 'backgroundListener', type: 'boolean' },
{ name: 'displayResidentHeader', type: 'boolean' },
{ name: 'currentResident', type: 'string', isOptional: true },
]
})
You can use a json field type for that I believe. https://nozbe.github.io/WatermelonDB/Advanced/AdvancedFields.html#json
Hi i have an array of objects like below,
const arr_obj = [
{
id: '1',
jobs: [
{
completed: false,
id: '11',
run: {
id: '6',
type: 'type1',
},
},
{
completed: true,
id: '14',
run: {
id: '17',
type: 'type1',
},
},
{
completed: false,
id: '12',
run: {
id: '7',
type: 'type2',
},
},
],
},
{
id: '2',
jobs: [
{
completed: true,
id: '13',
run: {
id: '8',
type: 'type2',
},
},
{
completed: true,
id: '16',
run: {
id: '9',
type: 'type1',
},
},
{
completed: true,
id: '61',
run: {
id: '19',
type: 'type1',
},
},
],
},
{
id: '3',
jobs: [
{
completed: false,
id:'111',
run: {
id: '62',
type: 'type1',
},
},
],
},
],
Now i have to get ids from arr_obj which matches this arr of ids
const arr_ids = ["1","2"]
and whose runs are of type "type1" and none of the jobs have completed: false
so the expected output is "2"
the below code works,
const filtered_arr_obj = arr_obj.filter(obj => {
if (arr_ids.includes(obj.id)) {
const jobs = obj.jobs.filter(job => job.run.type === "type1");
if (jobs.length > 0) {
return jobs.every(job => job.completed === true);
}
return false;
}
return false;
});
let filtered_ids = filtered_arr_obj.map(obj => obj.id);
console.log("filteredIds", filteredIds) //2
Now the problem is if there are no run and run of type "type" for job then i want to also filter the object which has no job run of type "type1"
so consider below example
const arr_obj = [
{
id: '1',
jobs: [
{
completed: false,
id: '11',
run: {
id: '6',
type: 'type1',
},
},
{
completed: true,
id: '14',
run: {
id: '17',
type: 'type1',
},
},
{
completed: false,
id: '12',
run: {
id: '7',
type: 'type2',
},
},
],
},
{
id: '2',
jobs: [
{
completed: true,
id: '13',
run: {
id: '8',
type: 'type2',
},
},
{
completed: true,
id: '16',
run: {
id: '9',
type: 'type1',
},
},
],
},
{
id: '3',
jobs: [
{
completed: false,
id: '111',
run: {
id: '62',
type: 'type1',
},
},
],
},
{
id: '4',
jobs: [
{
completed: true,
id: '121',
run: {
id: '66',
type: 'type2',
},
},
],
},
],
and
arr_ids = ["1", "2", "4"]
so the expected output is
["2", "4"]
here 4 is also selected because arr_ids has "4" and the object in arr_obj with id "4" has no job run with type "type1".
how can I modify the above code snippet to also include this condition? could someone help me with this? thanks.
I am new to programming and please do answer the question.
It Looks like when the id is present in the arr_ids and if the jobs length is <= 1 then we should filter it , else we are filtering the type1 jobs and checking whether all are completed .
const arr_obj = [
{
id: '1',
jobs: [
{
completed: false,
id: '11',
run: {
id: '6',
type: 'type1',
},
},
{
completed: true,
id: '14',
run: {
id: '17',
type: 'type1',
},
},
{
completed: false,
id: '12',
run: {
id: '7',
type: 'type2',
},
},
],
},
{
id: '2',
jobs: [
{
completed: true,
id: '13',
run: {
id: '8',
type: 'type2',
},
},
{
completed: true,
id: '16',
run: {
id: '9',
type: 'type1',
},
},
{
completed: true,
id: '61',
run: {
id: '19',
type: 'type1',
},
},
],
},
{
id: '3',
jobs: [
{
completed: false,
id:'111',
run: {
id: '62',
type: 'type1',
},
},
],
},
{
id: '4',
jobs: [
{
completed: true,
id: '121',
run: {
id: '66',
type: 'type2',
},
},
],
},
];
const arr_ids = ["1", "2", "4"];
const filtered_arr_obj = arr_obj.filter(obj => {
if (arr_ids.includes(obj.id)) {
if(obj.jobs.length <= 1 ){
return true;
}
else {
const type1jobs = obj.jobs.filter(job => job.run.type === "type1");
return type1jobs.every(job => job.completed );
}
}
return false;
});
let filtered_ids = filtered_arr_obj.map(obj => obj.id);
console.log("filteredIds", filtered_ids)
I have the following nested object:
export const mockData = {
sections: [
{
name: 'Client',
subSections: [
{
name: 'Client Detail'
}
]
},
{
name: 'Sales',
subSections: [
{
name: 'Overview',
subSections: [
{
name: 'Overview - A',
fields: [
{
key: 'firmName',
type: 'input',
templateOptions: {
label: 'Firm Name',
required: true
}
},
{
key: 'requestOption',
type: 'select',
templateOptions: {
label: 'Request Option',
required: true,
options: [
{ value: '1', label: '1' },
{ value: '2', label: '2' },
{ value: '3', label: '3' },
{ value: '4', label: '4' }
]
}
},
{
key: 'region',
type: 'select',
templateOptions: {
label: 'Region',
required: true,
options: [
{ value: '1', label: '1' },
{ value: '2', label: '2' },
{ value: '3', label: '3' },
{ value: '4', label: '4' },
{ value: '5', label: '5' }
]
}
}
]
}
]
}
]
}
]
};
and I would like to access the array from the fields property in order to render a form. Right now what I have is overviewA: FormlyFieldConfig[] = mockData.sections[1].subSections[0].subSections[0].fields, but I am facing the error of
Property 'subSections' does not exist on type '{ name: string; }'.
However, this error goes away when do a work-around and I add the property subSections to the nested object like so:
export const mockData = {
sections: [
{
name: 'Client',
subSections: [
{
name: 'Client Detail',
subSections: []
}
]
},
{
name: 'Sales',
subSections: [
{
name: 'Overview',
subSections: [
{
name: 'Overview - A', ...
I was wondering why
I was facing the issue without the work-around, and
How can I access the array from the property fields without the work-around?