Why is MongoDB not using the compound index for the query? - database

Question

Here are the compound index and single index I have for this Collection:
///db.Collection.getIndexes()
/* 1 */
{
"v" : 2,
"key" : {
"_id" : 1
},
"name" : "_id_",
"ns" : "service.Collection"
},
/* 2 */
{
"v" : 2,
"key" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"name" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"ns" : "service.Collection"
},
/* 3 */
{
"v" : 2,
"key" : {
"CreationTime" : 1
},
"name" : "CreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"ns" : "service.Collection"
}
The expected result is an IXSCAN using the FirstIdSecondIdCreationTime index:
///service.Collection.find({ FirstId: "771367b7-4bef-49ab-bda1-6230254c6349", ///SecondId: "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f" })
/// .projection({})
/// .sort({_id:-1}).hint("FirstIdSecondIdCreationTime").explain('executionStats')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "service.Collection",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"_id" : -1
},
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"indexName" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"FirstId" : [
"[MinKey, MaxKey]"
],
"SecondId" : [
"[MinKey, MaxKey]"
],
"CreationTime" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 5491,
"totalKeysExamined" : 856730,
"totalDocsExamined" : 856730,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 1,
"executionTimeMillisEstimate" : 5261,
"works" : 856734,
"advanced" : 1,
"needTime" : 856732,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"sortPattern" : {
"_id" : -1
},
"memUsage" : 432,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 1,
"executionTimeMillisEstimate" : 5201,
"works" : 856732,
"advanced" : 1,
"needTime" : 856730,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 5131,
"works" : 856731,
"advanced" : 1,
"needTime" : 856729,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 856730,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 856730,
"executionTimeMillisEstimate" : 820,
"works" : 856731,
"advanced" : 856730,
"needTime" : 0,
"needYield" : 0,
"saveState" : 6697,
"restoreState" : 6697,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"indexName" : "FirstIdSecondIdCreationTime",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 1,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"FirstId" : [
"[MinKey, MaxKey]"
],
"SecondId" : [
"[MinKey, MaxKey]"
],
"CreationTime" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 856730,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"indexDef" : {
"indexName" : "FirstIdSecondIdCreationTime",
"isMultiKey" : false,
"multiKeyPaths" : {
"FirstId" : [ ],
"SecondId" : [ ],
"CreationTime" : [ ]
},
"keyPattern" : {
"FirstId" : 1,
"SecondId" : 1,
"CreationTime" : -1
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"direction" : "forward"
}
}
}
}
}
but the actual result is a COLLSCAN that takes over 8000ms:
"event": {
"dataset": "mongodb.log",
"module": "mongodb"
},
"service": {
"type": "mongodb"
},
"message": "command service.Collection command: find { find: \"Collection\",
filter: { FirstId: \"771367b7-4bef-49ab-bda1-6230254c6349\", SecondId: \"3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f\" }, sort: { CreationTime: -1 }, limit: 1,
planSummary: COLLSCAN keysExamined:0 docsExamined:784787 hasSortStage:1 cursorExhausted:1 numYields:6175 nreturned:1 reslen:677
locks:{ Global: { acquireCount: { r: 12352 } }, Database: { acquireCount: { r: 6176 } }, Collection: { acquireCount: { r: 6176 } } } protocol:op_msg 8441ms",
"mongodb.docsExamined": 784787,
"fileset": {
"name": "log"
},
Why am I COLLSCANing instead of IXSCANing with the FirstIdSecondIDCreationTime compound index? Is there a way to change my index/ my query to speed up the query?
Per a suggestion in the comments, I've run explain("allPlansExecution").
///db.Collection.find({ FirstId: "771367b7-4bef-49ab-bda1-6230254c6349", ///SecondId: "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f" })
/// .projection({})
/// .sort({_id:-1}).explain('allPlansExecution')
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "service.Collection",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 5408,
"totalKeysExamined" : 856748,
"totalDocsExamined" : 856748,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"FirstId" : {
"$eq" : "771367b7-4bef-49ab-bda1-6230254c6349"
}
},
{
"SecondId" : {
"$eq" : "3bffb3cd-fb5e-43e5-abd1-e0b48c97f78f"
}
}
]
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 4862,
"works" : 856749,
"advanced" : 1,
"needTime" : 856747,
"needYield" : 0,
"saveState" : 6694,
"restoreState" : 6694,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 856748,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 856748,
"executionTimeMillisEstimate" : 1220,
"works" : 856749,
"advanced" : 856748,
"needTime" : 0,
"needYield" : 0,
"saveState" : 6694,
"restoreState" : 6694,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"_id" : 1
},
"indexName" : "_id_",
"isMultiKey" : false,
"multiKeyPaths" : {
"_id" : [ ]
},
"isUnique" : true,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "backward",
"indexBounds" : {
"_id" : [
"[MaxKey, MinKey]"
]
},
"keysExamined" : 856748,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
},
"allPlansExecution" : [ ]
}
}

Answer 1

The "FirstIdSecondIdCreationTime" index was not automatically considered because it was created with a collation, and the query is being run without a collation.
Use the .collation() cursor method to specify the same collation for the query that was used for the index.
The 5.5 second run time using that index is pretty slow as well.
You may see some improvement in that query if you create an index on {FirstId: 1, SecondId: 1, _id: 1} so that they query executor can use the index to meet the sort instead of an in-memory sort.

Answer 2

Can you please sort by leading indexes i.e firstId, secondId and creationTime in the same sequence and see index is used. it will give an idea whether leading indexes fields should be there in sort as well.