mongodb 2.6.3 use wrong index - mongodb-indexes

I have a query like below.Obviously the better index is "platform_1_keyuserAccountID_1_weiboAccountID_1_postTime_1",but mongodb choose "weiboAccountID_1",am i make something wrong?
db.inbox_weibo.find({platform: "sina", keyuserAccountID:"1665337047", weiboAccountID: "2271338624"}).explain(true);
{
"cursor" : "BtreeCursor weiboAccountID_1",
"isMultiKey" : false,
"n" : 1,
"nscannedObjects" : 1,
"nscanned" : 1,
"nscannedObjectsAllPlans" : 24,
"nscannedAllPlans" : 634425,
"scanAndOrder" : false,
"indexOnly" : false,
"nYields" : 1,
"nChunkSkips" : 0,
"millis" : 2912,
"indexBounds" : {
"weiboAccountID" : [
[
"2271338624",
"2271338624"
]
]
},
"allPlans" : [
{
"cursor" : "BtreeCursor weiboAccountID_1",
"isMultiKey" : false,
"n" : 1,
"nscannedObjects" : 1,
"nscanned" : 1,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"weiboAccountID" : [
[
"2271338624",
"2271338624"
]
]
}
},
{
"cursor" : "BtreeCursor platform_1_keyuserAccountID_1_dealed_1_operatorID_1_weiboAccountID_1_postTIme_-1",
"isMultiKey" : false,
"n" : 1,
"nscannedObjects" : 1,
"nscanned" : 62,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealed" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"operatorID" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"weiboAccountID" : [
[
"2271338624",
"2271338624"
]
],
"postTIme" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor platform_1_keyuserAccountID_1_weiboAccountID_1_postTime_-1",
"isMultiKey" : false,
"n" : 1,
"nscannedObjects" : 1,
"nscanned" : 1,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"weiboAccountID" : [
[
"2271338624",
"2271338624"
]
],
"postTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor platform_keyuserAccountID_dealed_postTime_weiboAccountID",
"isMultiKey" : false,
"n" : 1,
"nscannedObjects" : 1,
"nscanned" : 634331,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealed" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"postTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"weiboAccountID" : [
[
"2271338624",
"2271338624"
]
]
}
},
{
"cursor" : "BtreeCursor platform_1_keyuserAccountID_1_mid_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"mid" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor platform_1_keyuserAccountID_1_dealerID_1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealerID" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor distinctAccountAgg",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealed" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"operatorID" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"postTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor fansCountUndealed",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealed" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"fromType" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"fansCount" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"postTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"usePicture" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor retweetCountUndealed",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealed" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"fromType" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"sourceRetweetCount" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"postTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"usePicture" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor interactUndealed",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealed" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"fromType" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"userInteractionCount" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"postTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
],
"usePicture" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor platform_1_keyuserAccountID_1_dealed_1_dealTime_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealed" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"dealTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor platform_1_keyuserAccountID_1_dealerID_1_postTime_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealerID" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"postTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor platform_1_keyuserAccountID_1_postTime_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"platform" : [
[
"sina",
"sina"
]
],
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"postTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
},
{
"cursor" : "BtreeCursor keyuserAccountID_dealed_createTime_-1",
"isMultiKey" : false,
"n" : 0,
"nscannedObjects" : 2,
"nscanned" : 3,
"scanAndOrder" : false,
"indexOnly" : false,
"nChunkSkips" : 0,
"indexBounds" : {
"keyuserAccountID" : [
[
"1665337047",
"1665337047"
]
],
"dealed" : [
[
{
"$minElement" : 1
},
{
"$maxElement" : 1
}
]
],
"createTime" : [
[
{
"$maxElement" : 1
},
{
"$minElement" : 1
}
]
]
}
}

I don't think the other index is "better". It seems that both indexes (the one you would suggest and the one mongodb uses) return the queried object with "nscanned:1", so they are equally perfect for this query. I say that since "weiboAccountID" in your test dataset defines exactly the one record you are looking for, there is no need for other indexes in this case - mongodb will choose the first good fit. Try adding more records with this same exact weiboAccountID (if duplicated Ids are at all valid in your dataset) and re-run the query, see how it changes the execution.

Related

How to filter from array in a mongo document

I have a collection in mongoDB with documents like:
{
"_id" : ObjectId("some_id"),
"name" : "name",
"field1" : 123,
"field2" : 234,
"arr" : [
{
"a_num" : 3,
"arr" : [
"abc",
"def"
],
},
{
"a_num" : 36,
"arr" : [
"hi"
],
},
{
"a_num" : 34,
"arr" : [
"abc"
],
}
]
}
While I'm using find()
I don't want to get the arr elements where arr has only 1 element, that its value is "abc".
E.g, for the above document, I'd like to get:
{
"_id" : ObjectId("some_id"),
"name" : "name",
"field1" : 123,
"field2" : 234,
"arr" : [
{
"a_num" : 3,
"arr" : [
"abc",
"def"
],
},
{
"a_num" : 36,
"arr" : [
"hi"
],
},
]
}
Any idea how?
Thanks!
You'll need to use the aggregation framework with $filter, like so:
db.collection.aggregate([
{
$addFields: {
arr: {
$filter: {
input: "$arr",
cond: {
$ne: [
"$$this.arr",
[
"abc"
]
]
}
}
}
}
}
])

Why is MongoDB not using the compound index for the query?

Here are the compound index and single index I have for this Collection:
///db.Collection.getIndexes()
/* 1 */
{
"v" : 2,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "service.Collection"
},
/* 2 */
{
"v" : 2,
"key" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"name" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"ns" : "service.Collection"
},
/* 3 */
{
"v" : 2,
"key" : {
"CreationTime" : 1
},
"name" : "CreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"ns" : "service.Collection"
}
The expected result is an IXSCAN using the FirstIdSecondIdCreationTime index:
///service.Collection.find({ FirstId: "771367b7-4bef-49ab-bda1-6230254c6349", ///SecondId: "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f" })
/// .projection({})
/// .sort({_id:-1}).hint("FirstIdSecondIdCreationTime").explain('executionStats')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "service.Collection",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"indexName" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"FirstId" : [
"[MinKey, MaxKey]"
],
"SecondId" : [
"[MinKey, MaxKey]"
],
"CreationTime" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 5491,
"totalKeysExamined" : 856730,
"totalDocsExamined" : 856730,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1,
"executionTimeMillisEstimate" : 5261,
"works" : 856734,
"advanced" : 1,
"needTime" : 856732,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"_id" : -1
},
"memUsage" : 432,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 1,
"executionTimeMillisEstimate" : 5201,
"works" : 856732,
"advanced" : 1,
"needTime" : 856730,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 5131,
"works" : 856731,
"advanced" : 1,
"needTime" : 856729,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 856730,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 856730,
"executionTimeMillisEstimate" : 820,
"works" : 856731,
"advanced" : 856730,
"needTime" : 0,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"indexName" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"FirstId" : [
"[MinKey, MaxKey]"
],
"SecondId" : [
"[MinKey, MaxKey]"
],
"CreationTime" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 856730,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"indexDef" : {
"indexName" : "FirstIdSecondIdCreationTime",
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"direction" : "forward"
}
}
}
}
}
but the actual result is a COLLSCAN that takes over 8000ms:
"event": {
"dataset": "mongodb.log",
"module": "mongodb"
},
"service": {
"type": "mongodb"
},
"message": "command service.Collection command: find { find: \"Collection\",
filter: { FirstId: \"771367b7-4bef-49ab-bda1-6230254c6349\", SecondId: \"3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f\" }, sort: { CreationTime: -1 }, limit: 1,
planSummary: COLLSCAN keysExamined:0 docsExamined:784787 hasSortStage:1 cursorExhausted:1 numYields:6175 nreturned:1 reslen:677
locks:{ Global: { acquireCount: { r: 12352 } }, Database: { acquireCount: { r: 6176 } }, Collection: { acquireCount: { r: 6176 } } } protocol:op_msg 8441ms",
"mongodb.docsExamined": 784787,
"fileset": {
"name": "log"
},
Why am I COLLSCANing instead of IXSCANing with the FirstIdSecondIDCreationTime compound index? Is there a way to change my index/ my query to speed up the query?
Per a suggestion in the comments, I've run explain("allPlansExecution").
///db.Collection.find({ FirstId: "771367b7-4bef-49ab-bda1-6230254c6349", ///SecondId: "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f" })
/// .projection({})
/// .sort({_id:-1}).explain('allPlansExecution')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "service.Collection",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 5408,
"totalKeysExamined" : 856748,
"totalDocsExamined" : 856748,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 4862,
"works" : 856749,
"advanced" : 1,
"needTime" : 856747,
"needYield" : 0,
"saveState" : 6694,
"restoreState" : 6694,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 856748,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 856748,
"executionTimeMillisEstimate" : 1220,
"works" : 856749,
"advanced" : 856748,
"needTime" : 0,
"needYield" : 0,
"saveState" : 6694,
"restoreState" : 6694,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 856748,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
},
"allPlansExecution" : [ ]
}
}
The "FirstIdSecondIdCreationTime" index was not automatically considered because it was created with a collation, and the query is being run without a collation.
Use the .collation() cursor method to specify the same collation for the query that was used for the index.
The 5.5 second run time using that index is pretty slow as well.
You may see some improvement in that query if you create an index on {FirstId: 1, SecondId: 1, _id: 1} so that they query executor can use the index to meet the sort instead of an in-memory sort.
Can you please sort by leading indexes i.e firstId, secondId and creationTime in the same sequence and see index is used. it will give an idea whether leading indexes fields should be there in sort as well.

Working with Mongodb References inside array next to other content

I want to use the mongodb aggregation framework on 3 Collections that have to "come together" to one Query. The problem is, when I'm referencing to another collection with $lookup it deletes the other content of the array the reference is in.
Heres the collection my aggregation starts in (users):
{
"_id" : ObjectId("5c9bea89f4fe8c37175ade58"),
"kundennummer" : "000001",
"passwort" : "xxx",
"status" : "1",
"onlinestatus" : true,
"kontakt" : {
"email" : "test#test.net"
},
"thing" : [
{
"thing_id" : 2,
"onlinestatus" : false,
"status" : true,
"site" : [
{
"site_id" : 3,
"status": true
},
{
"site_id" : 4,
"status": true
}
],
"refs" : [
{
"thing_id" : 11,
"status" : true
},
{
"thing_id" : 22,
"status" : true
}
]
}
]
}
when I know want site to be extended by the content given in site collection like this:
{
"_id": 11,
"name": "test"
},
{
"_id": 22,
"name": "test2"
}
I tried to use $lookup and the status: true disappeared.
db.users.aggregate([{
$lookup:
{
from: "sites",
localField: "things.site.site_id",
foreignField: "_id",
as: "things.site"
}
}])
edit:
What I want to achieve is the following:
{
"_id" : ObjectId("5c9bea89f4fe8c37175ade58"),
"kundennummer" : "000001",
"passwort" : "xxx",
"status" : "1",
"onlinestatus" : true,
"kontakt" : {
"email" : "test#test.net"
},
"thing" : [
{
"thing_id" : 2,
"onlinestatus" : false,
"status" : true,
"site" : [
{
"site_id" : 11,
"status": true,
"name": "test"
},
{
"site_id" : 12,
"status": true,
"name": "test2"
}
],
"refs" : [
{
"thing_id" : 11,
"status" : true,
"name": "test"
},
{
"thing_id" : 12,
"status" : true,
"name": "test2"
}
]
}
]
}
To use $lookup on the embedded document in MongoDB aggregation you need to use $unwind. It will output a document for each element and then you can use $lookup. You will get results from sites as well with the help of the below query:
db.user.aggregate([
{
$unwind: "$thing"
},
{
$unwind: "$thing.site"
},
{
$unwind: "$thing.refs"
},
{
$lookup: {
from: 'sites',
let: { 'siteId': '$thing.site.site_id' },
pipeline: [
{
$match: {
$expr: {
$eq: ['$_id', '$$siteId']
}
}
},
],
as: 'siteObject'
}
},
{
$lookup: {
from: 'things',
let: { 'thingId': '$thing.refs.thing_id' },
pipeline: [
{
$match: {
$expr: {
$eq: ['$_id', '$$thingId']
}
}
},
],
as: 'thingObject'
}
},
{
$unwind: "$siteObject"
},
{
$unwind: "$thingObject"
},
{
$project: {
"_id" : 1,
"kundennummer" : 1,
"passwort" : 1,
"status" : 1,
"onlinestatus" : 1,
"kontakt" : 1,
"thing": [{
"thing_id" : "$thing.thing_id",
"onlinestatus" : "$thing.onlinestatus",
"status" : "$thing.status",
"site" : {
"site_id":"$thing.site.site_id",
"status":"$thing.site.status",
"siteName":"$siteObject.siteName",
},
"refs" : {
"thing_id":"$thing.refs.thing_id",
"status":"$thing.refs.status",
"siteName":"$thingObject.thingName",
},
}]
}
},
{
$unwind: "$thing"
},
{
$group: {
_id: {
"_id": "$_id",
"kundennummer": "$kundennummer",
"passwort": "$passwort",
"status":"$status",
"onlinestatus":"$onlinestatus",
"kontakt":"$kontakt"
},
thing: { $push: { thing_id: "$thing.thing_id", onlinestatus: "$thing.onlinestatus", status: "$thing.status",site:"$thing.site",refs: "$thing.refs" } }
}
}
])
This will result as below:
{
"_id" : {
"_id" : ObjectId("5c9bea89f4fe8c37175ade58"),
"kundennummer" : "000001",
"passwort" : "xxx",
"status" : "1",
"onlinestatus" : true,
"kontakt" : {
"email" : "test#test.net"
}
},
"thing" : [
{
"thing_id" : 2,
"onlinestatus" : false,
"status" : true,
"site" : {
"site_id" : ObjectId("5caf395e512ab5123bb4b0af"),
"status" : true,
"siteName" : "Moto"
},
"refs" : {
"thing_id" : ObjectId("5cb042ea512ab5123bb4b0b0"),
"status" : true,
"siteName" : "hi"
}
},
{
"thing_id" : 2,
"onlinestatus" : false,
"status" : true,
"site" : {
"site_id" : ObjectId("5caf395e512ab5123bb4b0af"),
"status" : true,
"siteName" : "Moto"
},
"refs" : {
"thing_id" : ObjectId("5cb042ea512ab5123bb4b0b1"),
"status" : true,
"siteName" : "world"
}
},
{
"thing_id" : 2,
"onlinestatus" : false,
"status" : true,
"site" : {
"site_id" : ObjectId("5caf395e512ab5123bb4b0ae"),
"status" : true,
"siteName" : "hello"
},
"refs" : {
"thing_id" : ObjectId("5cb042ea512ab5123bb4b0b0"),
"status" : true,
"siteName" : "hi"
}
},
{
"thing_id" : 2,
"onlinestatus" : false,
"status" : true,
"site" : {
"site_id" : ObjectId("5caf395e512ab5123bb4b0ae"),
"status" : true,
"siteName" : "hello"
},
"refs" : {
"thing_id" : ObjectId("5cb042ea512ab5123bb4b0b1"),
"status" : true,
"siteName" : "world"
}
}
]
}
Result is near by your desired result and i think you should not follow such schema it will make your every query more complex.

mongodb node.js nested array fields concatenation single variable result

I am using using this code inside model aggregation
{$project:{
//for brevity
}},
{$project:{
"employe_detail":{
"$map":{
"input":"$employe_detail",
"as":"names",
"in":{
"name":{"$concat":["$$names.first_name",
" ","$$names.other_names"," ",
"$$names.last_name"]}
}
}
}
}}
This is the result
{ "employe_detail" : [ { "name" : "Brian Smith" } ] }
{ "employe_detail" : [ { "name" : "Josh Clefton" } ] }
{ "employe_detail" : [ { "name" : "Treasure Dwayne" } ] }
when I try to extend the result fields output
{$project:{
//for brevity
}},
{$project:{
"employe_detail":{
"$map":{
"input":"$employe_detail",
"as":"names",
"in":{
"name":{"$concat":["$$names.first_name",
" ","$$names.other_names"," ",
"$$names.last_name"]},
"dept":"$$names.activity_year.activity_detail.dept",
"time_spent": "$$names.activity_year.activity_detail.activity_time_spent.time_spent",
"shift": "$$names.activity_year.activity_detail.activity_time_spent.shift"
}
}
}
}}
The result was
{ "employe_detail" : [ { "name" : "Brian Smith", "dept" : [ [ "spray", "smoothing", "assembling", "packaging" ] ],"time_spent" : [ [ [ 6 ], [ 15 ], [ 7 ], [ 8 ]] ], "shift" : [ [ [ "afternoon" ], [ "afternoon" ], [ "morning" ], [ "morning" ] ] ]} ] }
{ "employe_detail" : [ { "name" : "Josh Clefton", "dept" : [ [ "spray", "shining", "shaping", "smoothing"] ],"time_spent" : [ [ [ 5 ], [ ], [ 8 ], [ 10 ] ] ], "shift" : [ [ [ "afternoon" ], [ ], [ "night" ], [ "night" ] ] ] } ] }
{ "employe_detail" : [ { "name" : "Treasure Dwayne", "dept" : [ [ "spray", "shaping", "smoothing", "assembling" ] ], "time_spent" : [ [ [ 3 ], [ 9 ], [ 13 ], [ 9 ] ] ], "shift" : [ [ [ "morning" ], [ "morning" ], [ "morning" ], [ "morning" ] ] ] } ] }
Please how can I (at least) get something similar to this
{ "employe_detail" : [ { "name" : "Brian Smith",
{"dept" : "spray", "time_spent":6, "shift": "afternoon"},
{"dept" : "smoothing", "time_spent": 15 , "shift": "afternoon"},
{"dept" : "assembling", "time_spent": 7, "shift": "morning"},
{"dept" : "packaging", "time_spent": 8, "shift": "morning"}
}]}
{ "employe_detail" : [ { "name" : "Josh Clefton",
{"dept" : "spray", "time_spent":5, "shift": "afternoon"},
{"dept" : "shining", "time_spent": , "shift": },
{"dept" : "shaping", "time_spent": 8, "shift": "night"},
{"dept" : "smoothing", "time_spent": 10, "shift": "night"}
}]}
Preferably I would like to get a data to become a variable name i.e. for example
instead of time_spent as the variable name, it should be the data for dept that would be the variable name
{ "employe_detail" : [ { "name" : "Josh Clefton",
{"spray":5, "shift": "afternoon"},
{"shining": , "shift": },
{"shaping": 8, "shift": "night"},
{"smoothing": 10, "shift": "night"}
}]}
Please how can i achieve the above
UPDATE
This is sample data
{ "_id" : ObjectId("5bd548380a84d90b5c2bb416"), "details" : {"first_name" : "Brian", "other_names" : "Stone", "last_name" : "Smith", "gender" : "male", "date_of_birth" : ISODate("2009-03-05T00:00:00Z") }, "contact" : [ { "_id" : ObjectId("5bd548380a84d90b5c2bb417"), "residential_address" : "no 5 smith"} ], "health" : [ { "_id" : ObjectId("5bd548380a84d90b5c2bb419"), "illness_name" : "Cold", "sypmtom" : "Sneezing", "normal_treatment" : ""} ], "activity_year" : { "_id" : ObjectId("5bd548380a84d90b5c2bb41a"), "activity_detail" : [ { "_id" : ObjectId("5bd548380a84d90b5c2bb41b"), "dept" : "spray", "activity_time_spent" : [ ] }, { "_id" : ObjectId("5bd548390a84d90b5c2bb41c"), "dept" : "smoothing", "activity_time_spent" : [ ] }, { "_id" : ObjectId("5bd548390a84d90b5c2bb41e"), "dept" : "assembling", "activity_time_spent" : [ ] }, { "_id" : ObjectId("5bd548390a84d90b5c2bb420"), "dept" : "packaging", "activity_time_spent" : [ ] }, { "_id" : ObjectId("5bd54add0a84d90b5c2bb5c5"), "dept" : "shining", "activity_time_spent" : [ ] }, { "_id" : ObjectId("5bff847078c700209c1b515f"), "dept" : "shining", "activity_time_spent" : [ { "_id" : ObjectId("5c1d659083e7551854c1681d"), "time_spent" : 10, "shift" : "afternoon", }, { "_id" : ObjectId("5c1d659083e7551854c1681e"), "time_spent" : 10, "shift" : "afternoon"} ] } ]}, "__v" : 14 },
{ "_id" : ObjectId("5bd548bf0a84d90b5c2bb45b"), "details" : {"first_name" : "Treasure", "other_names" : "Bliss", "last_name" : "Dwayne", "gender" : "female", "date_of_birth" : ISODate("2010-10-28T00:00:00Z") }, "contact" : [ { "_id" : ObjectId("5bd548bf0a84d90b5c2bb45c"), "residential_address" : "no 5 smith"} ], "health" : [ {"_id" : ObjectId("5bd548bf0a84d90b5c2bb45e"), "illness_name" : "Cold", "sypmtom" : "Sneezing", "normal_treatment" : ""} ], "activity_year" : { "_id" : ObjectId("5bd548bf0a84d90b5c2bb45f"), "activity_detail" : [ { "dept" : "spray", "activity_time_spent" : [ ] }, { "dept" : "smoothing", "activity_time_spent" : [ { "_id" : ObjectId("5c200ccea72a3e11f895dc26"), "time_spent" : 3, "shift" : "morning"}, { "_id" : ObjectId("5c215d2eea2ebe1c8043ccde"), "time_spent" : 5, "shift" : "morning"} ] }, { "dept" : "packaging", "activity_time_spent" : [ { "_id" : ObjectId("5c203e75084852185c583e13"), "time_spent" : 9,"shift" : "afternoon"}, { "_id" : ObjectId("5c20f50f4feffe0b00e8e9f7"), "time_spent" : 18,"shift" : "afternoon"} ] }, { "dept" : "shining", "activity_time_spent" : [ { "_id" : ObjectId("5c200b38a72a3e11f895dc08"), "time_spent" : 9, "shift" : "night", }, { "_id" : ObjectId("5c215acdea2ebe1c8043ccc0"), "time_spent" : 9, "shift" : "night" } ] }] }, "__v" : 9 },
None of your expected outputs is a valid JSON document but I hope that below solution will guide you to something that suits your needs.
First of all to create document keys dynamically you have to use $arrayToObject operator which expects an array of two-element arrays as an input. So you can use $addFields operator to reshape your activity_year.activity_detail array into that kind of format. You can use $map to do that and I assume you need first time_spent ($arrayElemAt used to get first element). Then you can use $concatArrays in the next stage to combine that data with name , try:
db.col.aggregate([
{
$addFields: {
activities: {
$map: {
input: "$activity_year.activity_detail",
as: "activity",
in: {
$let: {
vars: {
first_time_spent: { $arrayElemAt: [ "$$activity.activity_time_spent", 0 ] }
},
in: [
[ "$$activity.dept", { $ifNull: [ "$$first_time_spent.time_spent", "" ] } ],
[ "shift", { $ifNull: [ "$$first_time_spent.shift", "" ] } ]
]
}
}
}
}
}
},
{
$project: {
employee_details: {
$concatArrays: [
[{ "name": { $concat: [ "$details.first_name", " ","$details.other_names", " ", "$details.last_name"] } }],
{
$map: {
input: "$activities",
as: "a",
in: { $arrayToObject: "$$a" }
}
}
]
}
}
}
])
Outputs the data in following format:
{
"_id" : ObjectId("5bd548bf0a84d90b5c2bb45b"),
"employee_details" : [
{
"name" : "Treasure Bliss Dwayne"
},
{
"spray" : "",
"shift" : ""
},
{
"smoothing" : 3,
"shift" : "morning"
},
{
"packaging" : 9,
"shift" : "afternoon"
},
{
"shining" : 9,
"shift" : "night"
}
]
}

MongoDB query with elemMatch for nested array data matching from inside objects [duplicate]

This question already has answers here:
Retrieve only the queried element in an object array in MongoDB collection
(18 answers)
Closed 4 years ago.
MongoDB query with elemMatch for nested array data matching from inside objects.
Here we have three project objects and each contains an array of exams with multiple objects.
Here when "project_id" : ObjectId("5b6959e3dc68d21c6613c82d") , The "exams" array contain two times "writer.user_id":ObjectId("5b6959e3dc68d21c6613c77d")
how could i get these two data filtered from the first project object ?
db.exams.insert([{
"project_id" : ObjectId("5b6959e3dc68d21c6613c82d"),
"exams" : [
{
"allotment_type" : "manual",
"exam_id" : 1,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77d"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 1,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 1,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
},
{
"allotment_type" : "manual",
"exam_id" : 2,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77e"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 2,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 2,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
},
{
"allotment_type" : "manual",
"exam_id" : 3,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77d"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 3,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 3,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
}]
},
{
"project_id" : ObjectId("5b6959e3dc68d21c6613c82e"),
"exams" : [
{
"allotment_type" : "manual",
"exam_id" : 1,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77d"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 1,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 1,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
},
{
"allotment_type" : "manual",
"exam_id" : 2,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77e"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 2,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 2,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
},
{
"allotment_type" : "manual",
"exam_id" : 3,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77c"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 3,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 3,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
}]
},
{
"project_id" : ObjectId("5b6959e3dc68d21c6613c82f"),
"exams" : [
{
"allotment_type" : "manual",
"exam_id" : 1,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77d"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 1,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 1,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
},
{
"allotment_type" : "manual",
"exam_id" : 2,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77e"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 2,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 2,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
},
{
"allotment_type" : "manual",
"exam_id" : 3,
"topic" : "something",
"word_count" : 5,
"key_word_density" : 2,
"writer" : {
"user_id" : ObjectId("5b6959e3dc68d21c6613c77c"),
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"editor" : {
"user_id" : 3,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
},
"manager" : {
"user_id" : 3,
"versions" : [
{
"id" : 1,
"file_path" : "abcd"
},
{
"id" : 2,
"file_path" : "abcd"
},
{
"id" : 3,
"file_path" : "abcd"
}
]
}
}]
}]);
You can try with $filter aggregation
db.collection.aggregate([
{ "$project": {
"exams": {
"$filter": {
"input": "$exams",
"as": "exam",
"cond": {
"$eq": [ "$$exam.writer.user_id", ObjectId("5b6959e3dc68d21c6613c77d") ]
}
}
}
}}
])

Resources