Partial results if a Vespa-query times out - vespa

If one document in the result-set is a huge document (several MBs), then:
The query times out (it seems that querying is fast - but creating snippets using summary: dynamic times out)
children: [...] contains only garbage information
Is one of these solutions possible?
If Vespa can't answer field: [...], it should at least answer the document-id, e.g. id: "id:post:post::12345" instead the id: "index/content/0/..."
Only the snippets of the huge document should time out (currently all children: [...] return "id": "index/content/...")
Example:
{
"root": {
"id": "toplevel",
"relevance": 1,
"fields": {
"totalCount": 60
},
"coverage": {
"coverage": 100,
"documents": 1427628,
"full": true,
"nodes": 2,
"results": 2,
"resultsFull": 2
},
"errors": [
{
"code": 12,
"summary": "Timed out",
"message": "timeout waiting for summaries from sc0.num0"
}
],
"children": [
{
"id": "index:content/0/939fc4e6c4c9576cd77a12f3",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/45ee850d113ea6e6c42f2f9f",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/b9fcf825622f9ec8c0ea4e4d",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/1e1c4a96617fa6571c3c5ce7",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/2550e74ff60e88710f1f4f26",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/d1dd0fb89cea4613132a76f6",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/92a8c4abe0e358e3dab36fbb",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/bc36fea47a7628648d89e1b0",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/804475ba69aaa691de930e10",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/0577e69b0824684948027fc3",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/1b1f1144aa80385aca9a6027",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/34f8ddeca987fe2d77069009",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/f1befbe5d773e19811790e11",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/af2e009dd5ab895b1e3f9ff1",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/862f42f5c322642198b1f267",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/6d1fb4c45beb641fe1ce17ec",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/f9797f6e747b33b992faffb3",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/6a3e15cfab492c4a428ad529",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/9eb6407921012c10deb70115",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/905360bc8da5ff1b0b452246",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/24277dd2eef4167cec1bcf8a",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/a207cf989e02b1443d662cf1",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/bae6cb8b6e9fbc1032cb1ee4",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/24b0cd6cd7251074a9dea836",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/95b0f46b8fa9d22d497b08fc",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/41d534f99d1a27f6a0d2032b",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/c13287b86341bfbddda08b3a",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/59874c4cbff45ac155da5b11",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/28a1e4e530064836716e8175",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/8ebdc951376c0a82cafee577",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/79bcd638b357820e6f8ac10d",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/866db8cd10611dbc898fe5ec",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/a1444ef7d27205b34c60ff25",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/c82d1c8b0a134413203dde58",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/29c349724f46f9c1a7448eab",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/96f0f6cb2789f521c71318ca",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/b5b8046218c23d4c08fea8a7",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/2d4f7cf4e3c8d57e8dc7b401",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/562104630c6a15a001b0718c",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/dac419d3bcc45e7422548e74",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/9a4bb96ea031b3891a7f8d12",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/9b67994ac2d440bd6ac69d19",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/a7dbbc02b8889271441dfdc4",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/acdc61985bbdc43ca5cfaf56",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/a1429d9aa68422e0384f682e",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/11b0377300156b23c5a893fa",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/6d8075a57eef79a9f7060989",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/565b12cee0eccded7f384fb1",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/93c46b96dccd2ed8e52c6ae4",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/26acfa4c314761921da624a4",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/c99f784fa9d31a93c7b72ef0",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/88099aad4415647d62d705d9",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/b80c189a7e125a17a04c703e",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/3dea2a9e7e13211c0706e846",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/f1bdfae364ff4ad2bfee6236",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/7cc8a3c625ce3a11383c6157",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/44ddd465721f9dbf66f15f64",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/53d09e41a0a488c9ceffd4d7",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/76b024d20a1a78e255420e80",
"relevance": 0,
"source": "content"
},
{
"id": "index:content/0/a4ddfd13bd4cace8984f90dc",
"relevance": 0,
"source": "content"
}
]
}
}

General:
It's not expected that it should take that much longer to generate snippets for large documents - how large are the problematic documents? How much time has passed before timeout? (You can find timing details by adding e.g &tracelevel=5 to the query.
It's most definitely not expected that snippets contains garbage information, could you show an example of what you see? Are you sure this isn't a client-side problem?
To your concrete questions:
The id is part of the summary so it cannot give you the full id out of the box if the summary request fails. You can do this yourself if you like by putting the id in a different summary class and requesting that first (from a searcher doing two "fill" calls for the different summary classes).
There is one summary request (typically for multiple id's) to each partition node. It would be possible to return what we have when you get close to the timeout but as far as I remember we don't do that for summary requests today.
Anyway, before considering workarounds let's find out the reason for this, I can't recall this having been a problem with any application before. I suggest you create an issue at https://github.com/vespa-engine/vespa/issues as it is more suitable for doing this.

Related

JOLT Transform from Prefix Soup to Nested

I´m receiving this JSON as part of a SQL Query and I want to create nested objects out of the composed properties. This can be done easily if it was 1 element, but the MYSQL query returns an array right at the root:
[
{
"idlicense": 1,
"StartDate": "2022-11-15 00:00:00.0",
"EndDate": "2022-11-29 00:00:00.0",
"MonthlySearchMax": 500,
"Customer_CustomerId": 0,
"Customer_Guid": "c24c1fa3-0388-4c08-b431-8d0f05fe263a",
"Customer_Name": "User",
"Customer_CustStartDate": "2022-11-15 00:00:00.0",
"Connector_ConnectorId": 0,
"Connector_Name": "connector0",
"Connector_Version": "1.01"
},
{
"idlicense": 2,
"StartDate": "2022-11-15 00:00:00.0",
"EndDate": "2022-11-29 00:00:00.0",
"MonthlySearchMax": 500,
"Customer_CustomerId": 0,
"Customer_Guid": "c24c1fa3-0388-4c08-b431-8d0f05fe263a",
"Customer_Name": "User",
"Customer_CustStartDate": "2022-11-15 00:00:00.0",
"Connector_ConnectorId": 1,
"Connector_Name": "connector1",
"Connector_Version": "1.01"
}
]
I´m trying to create a nested JSON with JOLT on NIFI but can´t find the right format looking at the available examples and seems to be simple. I need my final JSON to look like this:
{
"Licenses": [
{
"idLicense": "1",
"Customer": {
"CustomerId": "0",
"Guid": "c24c1fa3-0388-4c08-b431-8d0f05fe263a",
"Name": "User",
"CustStartDate": "2022-11-15 00:00:00.0"
},
"Connector": {
"ConnectorId": "0",
"Name": "Connector0",
"Version": "1.01"
}
},
{
"idLicense": "2",
"Customers": {
"CustomerId": "1",
"Guid": "c24c1fa3-0388-4c08-b431-8d0f05fe263a",
"Name": "User",
"CustStartDate": "2022-11-15 00:00:00.0"
},
"Connector": {
"ConnectorId": "1",
"Name": "Connector1",
"Version": "1.01"
}
}
]
}
So far I´ve done this JOLT transform:
{
"idlicense": [1,2],
"Customer": {
"CustomerId": [0,0],
"Guid": ["c24c1fa3-0388-4c08-b431-8d0f05fe263a","c24c1fa3-0388-4c08-b431-8d0f05fe263a"],
"Name": ["User","User"],
"CustStartDate": ["2022-11-15 00:00:00.0","2022-11-15 00:00:00.0"]
},
"Connector": {
"ConnectorId": [0,1],
"Name": ["Conector0","Connector1"],
"Version": ["1.01","1.01"]
}
}
thank you for your support!
You can use this shift transformation spec
[
{
"operation": "shift",
"spec": {
"*": {
"idlicense": "Licenses[&1].&",
"*_*": "Licenses[&1].&(0,1).&(0,2)"
}
}
}
]
where &(0,1) represents before, &(0,2) after underscore character

Angular - get data from Object Array

Good morning, can anyone advise me? I'm pretty clueless. I've already spent a few hours on it and I don't know how to solve this.
The data is fictitious and the original JSON is far more complex.
JSON
{
"main": [
[
{
"type": "dasdasdasd",
"id": 5,
"content": {
"title": "adadadsad",
"items": [
{
"date": "2012-02-02T11:23:00Z",
"id": 12,
"name": "test",
"isEnabled": false,
"isHighlited": false,
"images": {
"lists": {
"small": [
{
"id": 18,
"position": 0,
"titleImage": true,
"url": "",
"thumbnailReady": true
}
],
"original": [
{
"id": 19,
"position": 0,
"titleImage": true,
"url": "",
"thumbnailReady": true
}
],
"large": [
{
"id": 22,
"position": 0,
"titleImage": true,
"url": "",
"thumbnailReady": true
}
],
"medium": [
{
"id": 23,
"position": 0,
"titleImage": true,
"url": "",
"thumbnailReady": true
}
]
}
},
"enum": "LINIE",
"url": "https://test.com"
}
]
}
}
]
],
"second": [
]
}
How can I get from main -> content -> items -> images -> url?
More precisely each images and from that url.
Thank you very much in advance for your help.
It looks like your main array has an array as 1st element. If that is correct, you take the first element of main, and then loop over that.
let jsonObj = {
"main": [
[
{
"type": "dasdasdasd",
"id": 5,
"content": {
"title": "adadadsad",
"items": [
{
"date": "2012-02-02T11:23:00Z",
"id": 12,
"name": "test",
"isEnabled": false,
"isHighlited": false,
"images": {
"lists": {
"small": [
{
"id": 18,
"position": 0,
"titleImage": true,
"url": "",
"thumbnailReady": true
}
],
"original": [
{
"id": 19,
"position": 0,
"titleImage": true,
"url": "",
"thumbnailReady": true
}
],
"large": [
{
"id": 22,
"position": 0,
"titleImage": true,
"url": "",
"thumbnailReady": true
}
],
"medium": [
{
"id": 23,
"position": 0,
"titleImage": true,
"url": "",
"thumbnailReady": true
}
]
}
},
"enum": "LINIE",
"url": "https://test.com"
}
]
}
}
]
],
"second": [
]
}
let smallImg = "", originalImg = "", largeImg = "", mediumImg = "";
jsonObj.main[0].forEach(mainItem => {
mainItem.content.items.forEach(item => {
smallImg = item.images.lists.small[0].url;
originalImg = item.images.lists.original[0].url;
largeImg = item.images.lists.large[0].url;
mediumImg = item.images.lists.medium[0].url;
});
});
console.log(smallImg);
console.log(originalImg);
console.log(largeImg);
console.log(mediumImg);

Avoid using arrays (and [0]) in multidimensional JSON object

I'm storing data in a JSON file and need different objects/arrays. So far I am using the following structure:
data= [
{
"savedRuns": [
{
"id": 1,
"name": "Run 1"
},
{
"id": 2,
"name": "Run 2"
},
{
"id": 3,
"name": "Run 3"
}
]
},
{
"groups": [
{
"id": 1,
"name": "g1"
},
{
"id": 2,
"name": "g2"
},
{
"id": 3,
"name": "g3"
}
]
},
{
"locations": [
{
"id": 1,
"name": "home"
},
{
"id": 2,
"name": "work"
},
{
"id": 3,
"name": "school"
}
]
}
]
I would like to access the data in the file in an easy way, for instance:
console.log(data.savedRuns)
console.log(data.locations)
Unfortunately this returns undefined and the only way to access the data is:
console.log(data[0].savedRuns);
console.log(data[2].locations);
Since I don't necessarily know the position of the object, I would like to avoid that. If there a way around this, or a different structure to adopt for my file?
Link to JSFiddle
You have each attribute in a separate object, stored in an array. Get rid of the outer array and store the attributes in one object, then use data.savedRuns.
Thanks to Marks answer I could get the right syntax. Posting below in case of interest to others.
data = {
"savedRuns": [{
"id": 1,
"name": "Run 1"
},
{
"id": 2,
"name": "Run 2"
},
{
"id": 3,
"name": "Run 3"
}
],
"groups": [{
"id": 1,
"name": "g1"
},
{
"id": 2,
"name": "g2"
},
{
"id": 3,
"name": "g3"
}
],
"locations": [{
"id": 1,
"name": "home"
},
{
"id": 2,
"name": "work"
},
{
"id": 3,
"name": "school"
}
]
}

How to sort parent document by child documents lowest price in solr

I am new in solr i want to sort parent document by child documents lowest price . Please see below mentioned sample data.
{
"parentID": 1,
"children": {
"numFound": 2,
"start": 0,
"docs": [
{
"type": "child",
"parentID": 1,
"ID": "1",
"price": 189.7
},
{
"type": "child",
"parentID": 1,
"ID": "2",
"price": 933.1
}
]
} }, {
"parentID": 2,
"children": {
"numFound": 2,
"start": 0,
"docs": [
{
"type": "child",
"parentID": 2,
"ID": "1",
"price": 1180.0
},
{
"type": "child",
"parentID": 2,
"ID": "2",
"price": 238.0
}
]
} }, {
"parentID": 3,
"children": {
"numFound": 1,
"start": 0,
"docs": [
{
"type": "child",
"parentID": 3,
"ID": "1",
"price": 1442.81,
},
{
"type": "child",
"parentID": 3,
"ID": "2",
"price": 42.81,
}
]
} }, {
"parentID": 4,
"children": {
"numFound": 1,
"start": 0,
"docs": [
{
"type": "child",
"parentID": 4,
"ID": "1",
"price": 1140.8,
}
]
} }
And i want to result below mentioned format
{
"parentID": 3,
"children": {
"numFound": 1,
"start": 0,
"docs": [
{
"type": "child",
"parentID": 3,
"ID": "1",
"price": 1442.81,
},
{
"type": "child",
"parentID": 3,
"ID": "2",
"price": 42.81,
}
]
}
},
{
"parentID": 1,
"children": {
"numFound": 2,
"start": 0,
"docs": [
{
"type": "child",
"parentID": 1,
"ID": "1",
"price": 189.7
},
{
"type": "child",
"parentID": 1,
"ID": "2",
"price": 933.1
}
]
}
},
{
"parentID": 2,
"children": {
"numFound": 2,
"start": 0,
"docs": [
{
"type": "child",
"parentID": 2,
"ID": "2",
"price": 238.0
},
{
"type": "child",
"parentID": 2,
"ID": "1",
"price": 1180.0
}
]
}
},
{
"parentID": 4,
"children": {
"numFound": 1,
"start": 0,
"docs": [
{
"type": "child",
"parentID": 4,
"ID": "1",
"price": 1140.8,
}
]
}
}
I tried below mentioned link it's not worked for my issue.
how to sort parent documents by child attributes in solr
sort={!parent which=type:parent score=max v=’+type:child +{!func}price’} desc

angular ui tree for complex json

How to create angular ui tree for complex json?
[
{
"name": "bmw",
"model": "x3",
"specs": [
{
"name": "bmw",
"maxspeed": 350,
"minspeed": 0,
"model": "x3",
"type": "car"
},
{
"value": 30,
"name": "bmw",
"maxspeed": 350,
"minspeed": 0,
"model": "x3",
"type": "car"
}
],
"available": true,
"price": 50
},
{
"name": "audi",
"model": "r8",
"specs": [
{
"color": [
{
"value": "red",
"price": 100
},
{
"value": [
"white",
"black"
],
"price": 200
}
],
"maxspeed": 330,
"minspeed": 0,
"type": "car"
},
{
"color": [
{
"value": "blue",
"price": 300
},
{
"value": [
"yellow",
"gold"
],
"price": 500
}
],
"maxspeed": 330,
"minspeed": 0,
"type": "car"
}
],
"available": true
}
]

Resources