Create mulitple records from array-of-arrays within Postgres JSON object - arrays

Within my Postgres database, I have a column (type=JSONB) that contains an API response that represents someone response to a Survey Monkey survey. Within this JSON object, there is an array called pages that contains at least 1 item, and something more than 1 item. Then within pages, there is another array called questions. This array also contains at least 1 item, and more often more than 1 item.
I am trying to write a Postgres query to extract the responses from all the questions seen within pages and questions. The final format should be 1 record per question. If the survey has 5 questions, that would create 5 rows. If the survey has 100 questions, across 5 pages, that would mean 100 rows.
+────────────+────────────────────────────────────────────+──────────────────────+
| id | heading | simple_text |
+────────────+────────────────────────────────────────────+──────────────────────+
| 775249149 | I am confident that I am the best. | Somewhat Agree |
| 775249153 | I currently have the skills to be amazing | Somewhat Agree |
| 775249166 | How long until I win the lottery? | 6 to 12 months more |
+────────────+────────────────────────────────────────────+──────────────────────+
I've been trying to use the function json_array_elements, but I'm experiencing some errors when trying to pass in an array-of-arrays.
{
"id": "12345",
"href": "https://api.surveymonkey.com",
"pages": [
{
"id": "142250690",
"questions": [
{
"id": "775249149",
"family": "matrix",
"answers": [
{
"row_id": "5133514018",
"choice_id": "5133514023",
"simple_text": "Somewhat Agree",
"choice_metadata": {
"weight": "5"
}
}
],
"heading": "I am confident that I am the best.",
"subtype": "rating"
},
{
"id": "775249153",
"family": "matrix",
"answers": [
{
"row_id": "5133514112",
"choice_id": "5133514117",
"simple_text": "Somewhat Agree",
"choice_metadata": {
"weight": "5"
}
}
],
"heading": "I currently have the skills to be amazing",
"subtype": "rating"
},
{
"id": "775249166",
"family": "matrix",
"answers": [
{
"row_id": "5133514278",
"choice_id": "5133514280",
"simple_text": "6 to 12 months more",
"choice_metadata": {
"weight": "2"
}
}
],
"heading": "How long until I win the lottery?",
"subtype": "rating"
}
]
}
],
"survey_id": "123456789",
"total_time": 29,
"date_created": "2022-07-25T23:08:36+00:00",
"recipient_id": "",
"date_modified": "2022-07-25T23:09:06+00:00",
"email_address": "",
"collection_mode": "default",
"response_status": "completed",
}

Try to use json_array_elements() twice:
http://sqlfiddle.com/#!17/9eecb/94455
select v->'id', v->'heading', v->'subtype', v->'answers'->0->'simple_text' from (
select json_array_elements(value->'questions')::jsonb as v from json_array_elements('{
"id": "12345",
"href": "https://api.surveymonkey.com",
"pages": [
{
"id": "142250690",
"questions": [
{
"id": "775249149",
"family": "matrix",
"answers": [
{
"row_id": "5133514018",
"choice_id": "5133514023",
"simple_text": "Somewhat Agree",
"choice_metadata": {
"weight": "5"
}
}
],
"heading": "I am confident that I am the best.",
"subtype": "rating"
},
{
"id": "775249153",
"family": "matrix",
"answers": [
{
"row_id": "5133514112",
"choice_id": "5133514117",
"simple_text": "Somewhat Agree",
"choice_metadata": {
"weight": "5"
}
}
],
"heading": "I currently have the skills to be amazing",
"subtype": "rating"
},
{
"id": "775249166",
"family": "matrix",
"answers": [
{
"row_id": "5133514278",
"choice_id": "5133514280",
"simple_text": "6 to 12 months more",
"choice_metadata": {
"weight": "2"
}
}
],
"heading": "How long until I win the lottery?",
"subtype": "rating"
}
]
},
{
"id": "142250690",
"questions": [
{
"id": "775249199",
"family": "matrix",
"answers": [
{
"row_id": "5133514278",
"choice_id": "5133514280",
"simple_text": "6 to 12 months more",
"choice_metadata": {
"weight": "2"
}
}
],
"heading": "qweqweqweqwqqqq?",
"subtype": "rating"
}
]
}
],
"survey_id": "123456789",
"total_time": 29,
"date_created": "2022-07-25T23:08:36+00:00",
"recipient_id": "",
"date_modified": "2022-07-25T23:09:06+00:00",
"email_address": "",
"collection_mode": "default",
"response_status": "completed"
}'::jsonb::json->'pages')
) t;
UPDATE
If you need to process data from table contained described json rows, you need something like this:
http://sqlfiddle.com/#!17/16b65/1

Related

Select random id from list in jq and update value

I have some sample payload that I am going to be receiving, it looks like this:
[
{
"Id": "9",
"Line": [
{
"Amount": 100,
"Description": "Weekly Gardening Service",
"DetailType": "SalesItemLineDetail",
"Id": "1",
"LineNum": 1,
"SalesItemLineDetail": {
"ItemAccountRef": {
"name": "Landscaping Services",
"value": "45"
},
"ItemRef": {
"name": "Gardening",
"value": "6"
},
"Qty": 4,
"TaxCodeRef": {
"value": "TAX"
},
"UnitPrice": 25
}
},
{
"Amount": 100,
"DetailType": "SubTotalLineDetail",
"SubTotalLineDetail": {}
}
]
},
{
"Id": "10",
"Line": [
{
"Amount": 140,
"Description": "Weekly Gardening Service",
"DetailType": "SalesItemLineDetail",
"Id": "1",
"LineNum": 1,
"SalesItemLineDetail": {
"ItemAccountRef": {
"name": "Landscaping Services",
"value": "45"
},
"ItemRef": {
"name": "Gardening",
"value": "6"
},
"Qty": 4,
"TaxCodeRef": {
"value": "NON"
},
"UnitPrice": 35
}
},
{
"Amount": 35,
"Description": "Pest Control Services",
"DetailType": "SalesItemLineDetail",
"Id": "2",
"LineNum": 2,
"SalesItemLineDetail": {
"ItemAccountRef": {
"name": "Pest Control Services",
"value": "54"
},
"ItemRef": {
"name": "Pest Control",
"value": "10"
},
"Qty": 1,
"TaxCodeRef": {
"value": "NON"
},
"UnitPrice": 35
}
},
{
"Amount": 175,
"DetailType": "SubTotalLineDetail",
"SubTotalLineDetail": {}
}
]
}
]
These I know are valid and I need to cross reference them, by id, in another payload I am receiving. But, the data I am receiving I can't assume to have valid ID's.
So, I want to take all the valid Ids from above, and shove them, randomly, into the sample data I have, that looks like this ($.invoices[].qbId):
[
{
"id": "fb2430c5-5970-46b0-9947-aaa0b9f177bb",
"invoices": [
{
"description": "2022-02-03 - 179",
"dueDate": "2022-02-03T22:51:10.206Z",
"id": "6f904b18-71c6-4fec-a016-7452f6a6b1dc",
"invoiceDate": "2022-02-03T22:51:10.347Z",
"openBalance": 200,
"paidAmount": 200,
"qbId": "1",
"totalAmount": 212
}
]
},
{
"id": "fa5b77b5-bfd4-4178-ac31-386ec83f530c",
"invoices": [
{
"description": "2022-01-12 - 95",
"dueDate": "2022-01-12T14:08:26.219Z",
"id": "05a58be3-4396-4c15-b9c2-ece68cb2b3fb",
"invoiceDate": "2022-01-12T14:08:26.399Z",
"openBalance": 7.33,
"paidAmount": 7.33,
"qbId": "",
"totalAmount": 7.33
},
{
"description": "2022-01-12 - 95",
"dueDate": "2022-01-12T14:08:26.219Z",
"id": "91f5ecd0-e18d-4029-8745-143323e02007",
"invoiceDate": "2022-01-12T14:08:26.580Z",
"openBalance": 53.13,
"paidAmount": 53.13,
"qbId": "",
"totalAmount": 53.13
}
]
}
]
this jq will get me my ids jq '.QueryResponse.Invoice | map(.Id)' which can be readily consumed by jq. The question now is (and this is what I don't know) how to randomly choose from this array and update the sample payload:
jq 'map(. + {
invoices : .invoices | map(. + {qbId: ??random here })
})
'
If I understood correctly, you want to replace each id field (spelling may differ, sometimes it's Id) with a randomly generated id string.
This solution first extracts the paths of all such id fields (in various spellings) using jq, then iterates over the result in the shell, using uuidgen to generate an id for each, which is fed into another jq call which uses setpath to change the value at the paths saved to the ids generated:
file="input.json"
jq -c '
paths(.. | scalars) | select(.[-1] == ("id", "Id", "ID")) | tojson
' "$file" |
while read -r json; do printf '["%s",%s]\n' "$(uuidgen)" "$json"; done |
jq -n --argfile file "$file" '
reduce inputs as [$id,$json] ($file; setpath($json | fromjson; $id))
'
[
{
"Id": "10162eb7-29ba-4b60-ad20-e5b1133eca63",
"Line": [
{
"Amount": 100,
"Description": "Weekly Gardening Service",
"DetailType": "SalesItemLineDetail",
"Id": "272832df-a8f5-4877-92de-1545150afc33",
"LineNum": 1,
"SalesItemLineDetail": {
"ItemAccountRef": {
"name": "Landscaping Services",
"value": "45"
},
"ItemRef": {
"name": "Gardening",
"value": "6"
},
"Qty": 4,
"TaxCodeRef": {
"value": "TAX"
},
"UnitPrice": 25
}
},
{
"Amount": 100,
"DetailType": "SubTotalLineDetail",
"SubTotalLineDetail": {}
}
]
},
{
"Id": "190b0e50-e007-46a4-b1ca-c3efb762629c",
"Line": [
{
"Amount": 140,
"Description": "Weekly Gardening Service",
"DetailType": "SalesItemLineDetail",
"Id": "f7067227-56d4-4849-873a-3ee5c336999e",
"LineNum": 1,
"SalesItemLineDetail": {
"ItemAccountRef": {
"name": "Landscaping Services",
"value": "45"
},
"ItemRef": {
"name": "Gardening",
"value": "6"
},
"Qty": 4,
"TaxCodeRef": {
"value": "NON"
},
"UnitPrice": 35
}
},
{
"Amount": 35,
"Description": "Pest Control Services",
"DetailType": "SalesItemLineDetail",
"Id": "181d7c6b-0afa-4f44-a568-2c482fc5c285",
"LineNum": 2,
"SalesItemLineDetail": {
"ItemAccountRef": {
"name": "Pest Control Services",
"value": "54"
},
"ItemRef": {
"name": "Pest Control",
"value": "10"
},
"Qty": 1,
"TaxCodeRef": {
"value": "NON"
},
"UnitPrice": 35
}
},
{
"Amount": 175,
"DetailType": "SubTotalLineDetail",
"SubTotalLineDetail": {}
}
]
}
]
This shows how to select elements at random from an array, assuming a bash or sufficiently bash-like environment:
#!/bin/bash
< /dev/urandom tr -cd '0-9' | fold -w 1 | jq -MRnc '
# Output: a prn in range(0;$n) where $n is `.`
def prn:
if . == 1 then 0
else . as $n
| ([1, (($n-1)|tostring|length)]|max) as $w
| [limit($w; inputs)] | join("") | tonumber
| if . < $n then . else ($n | prn) end
end;
# Input: an array
# Output: an array, being a selection of $k elements from . chosen at random without replacement
def prns($k):
if $k <= 0 then []
else . as $in
| length as $n
| if $k > $n then "no can do" | error
else ($n|prn) as $ix
| [$in[$ix]] + (($in[0:$ix] + $in[$ix+1:])|prns($k-1))
end
end;
# Two illustrations
# Three from range(0,10) (with replacement):
[range(0;10) | ( ["a", "b", "c"] | .[length|prn]) ],
# Three from an array, without replacement:
([range(0;10)] | prns(3))
'

How to get a value in one mongodb collection and use that value to update another document in another collection order & inventory system

Hello I have been stuck for weeks trying to figure how to create a order & inventory system for a project I am working on. I don't know how to properly ask this but my problem is when a user adds items to their cart > I store the order details in a orders collection in mongodb > I then need to figure out how to subtract the quantity of the items in a customers order from my inventory collection. How can I do this with mongodb, Python
This is the document created when a customer places an order
{
"_id": "5eca94b4f56331fd9eade681",
"ordernumber": 343,
"order": {
"order_details": [
{
"customer_info": [
{
"first_name": "John",
"last_name": "Doe",
"email": "email#email.com"
}
],
"shipping_details": [
{
"shipping_address": "Test Address",
"shipping_zip": "12345",
"shippingl_city": "Test city",
"shipping_country": "USA"
}
],
"products_ordered": [
{
"variant_id": "a",
"product_name": "red shirt",
"price": 30,
"quantity": 2,
"image": "imageurl",
"size": "Small"
},
{
"variant_id": "f",
"product_name": "Blue Jeans",
"price": 20,
"quantity": 3,
"image": "imageurl",
"size": "Large"
}
]
}
]
}
}
These are the products in my inventory collection I want inventory order quantity subtracted by the quantity a customer purchased
{
"_id": "5eca0ff4898b8f30a9fee5e5",
"product_id": 1,
"product_name": "red shirt",
"category": "shirts",
"price": 30,
"status": "instock",
"description": "nice red shirt",
"alt": "string",
"images": [
"imageUrl1",
"imageUrl2"
],
"variants": [
{
"Small": [
{
"variant_id": "a",
"inventory": 30
}
],
"Medium": [
{
"variant_id": "b",
"inventory": 10
}
],
"Large": [
{
"variant_id": "c",
"inventory": 10
}
]
}
]
}
{
"_id": "5eca108f898b8f30a9fee5e6",
"product_id": 2,
"product_name": "blue jeans",
"category": "jeans",
"price": 20,
"status": "instock",
"description": "nice blue jeans",
"alt": "string",
"images": [
"ImageURL"
],
"variants": [
{
"Small": [
{
"variant_id": "d",
"inventory": 100
}
],
"Medium": [
{
"variant_id": "e",
"inventory": 150
}
],
"Large": [
{
"variant_id": "f",
"inventory": 70
}
] }
]
}
I would suggest to do it along with the service which creates the order.
I would also like to suggest to refactor the db structure a bit as it would be harder to maintain this in a larger scale.
Because currently we would have to write something like
for ordered_product in products_ordered:
query = { "product_name": ordered_product.get("product_name") }
inventory_product = inventory_collection.find_one(query)
product_id = inventory_product["_id"]
existing_count = inventory_product["variants"][0][ordered_product.size][0]["inventory"]
inventory_product["variants"][0][ordered_product["size"]][0]["inventory"] = existing_count - ordered_product["quantity"]
inventory_collection.update_one({ "_id": product_id }, { "$set": inventory_product })
I have hardcoded the index values of the list. You could use filter() to filter out the variant and size you need.
This code definitely seems messy to me.
Of course you could refactor this code by splitting it into functions inside the model file itself, but I would suggest to refactor the db structure for better scalability.
May be you could move the variants to a seperate collection and use the product_id as a link. You have to think this through before getting on with the code.
Hope this helps.

how to select value from json file which find another value matches use jq

I have been struggling with json stuff. I want to find running state's href. How can I do that with jq or another like bash-style?
Here is my curl output:
{
"relations": {
"total": 9,
"link": [
{
"href": "https://vro:8281/vco/api/workflows/6433f56f-13b7-46a7-a9ec-a3e38c7ff69d/",
"rel": "up"
},
{
"href": "https://vro:8281/vco/api/workflows/6433f56f-13b7-46a7-a9ec-a3e38c7ff69d/executions/",
"rel": "add"
},
{
"attributes": [
{
"value": "8f961082-cccc-412f-9244-16ba5b949dbe",
"name": "id"
},
{
"value": "2019-09-28T17:20:40.691-01:00",
"name": "startDate"
},
{
"value": "2019-09-28T17:20:43.949-01:00",
"name": "endDate"
},
{
"value": "completed",
"name": "state"
},
{
"value": "test",
"name": "name"
},
{
"name": "currentItemDisplayName"
}
],
"href": "https://vro:8281/vco/api/workflows/6433f56f-13b7-46a7-a9ec-a3e38c7ff69d/executions/8f961082-cccc-412f-9244-16ba5b949dbe/",
"rel": "down"
},
{
"attributes": [
{
"value": "b28832cb-2a97-4ec8-848f-35fec95eb867",
"name": "id"
},
{
"value": "2019-09-28T17:21:04.643-01:00",
"name": "startDate"
},
{
"value": "running",
"name": "state"
},
{
"value": "test",
"name": "name"
},
{
"name": "currentItemDisplayName"
}
],
"href": "https://vro:8281/vco/api/workflows/6433f56f-13b7-46a7-a9ec-a3e38c7ff69d/executions/b28832cb-2a97-4ec8-848f-35fec95eb867/",
"rel": "down"
}
}
If I talk with example I want to find href key is https://vro:8281/vco/api/workflows/6433f56f-13b7-46a7-a9ec-a3e38c7ff69d/executions/b28832cb-2a97-4ec8-848f-35fec95eb867/ by way of running key.
After the sample JSON has been fixed, the following jq query:
.relations.link[]
| select( has("attributes") )
| select( any(.attributes[]; .value=="running" and .name == "state") )
| .href
produces:
"https://vro:8281/vco/api/workflows/6433f56f-13b7-46a7-a9ec-a3e38c7ff69d/executions/b28832cb-2a97-4ec8-848f-35fec95eb867/"
In two lines (i.e., with one pipe)
.relations.link[]
| select(any(.attributes[]?; .value=="running" and .name == "state")).href

Deleting Object from Nested Array MongoDB

I am trying to delete the object with the id: "2019-08-22T04:53:11.357Z" from this users portfolio array.
I have searched for the correct query filter to help me delete the object but nothing seems to be working!
Here is the data we are working with. There is currently only one user
(InStock) in the system but more will be added in the future.
[
{
"id": "MTg4MDU3ODM2MDk2NDU0NjU3",
"name": "InStock",
"balance": 7760,
"portfolio": [
{
"id": "2019-08-22T04:15:22.998Z",
"name": "Jordan Shoe",
"size": "10.5",
"price": 150
},
{
"id": "2019-08-22T04:36:37.836Z",
"name": "Nike Tee",
"size": "M",
"price": 35
},
{
"id": "2019-08-22T04:53:11.357Z",
"name": "Adidas Shoe",
"size": "8.5",
"price": 100
}
],
"history": [
]
}
]
and here is what I was trying using what I've seen in other solutions.
db.collection(collectionName).updateOne({"id": "MTg4MDU3ODM2MDk2NDU0NjU3"}, {$pull : {"portfolio": {"id": "2019-08-22T04:36:37.836Z"}}})
I am looking for the line to remove the Adidas Shoe object from InStock's portfolio array.
Try this query:
db.test1.updateOne({"_id" : ObjectId("5d5e7a291b761bfc0420e580")},
{$pull: {"portfolio": {"name": "Adidas Shoe"}}} )
After execution:
{
"_id" : ObjectId("5d5e7a291b761bfc0420e580"),
"portfolio" : [
{
"id" : "2019-08-22T04:15:22.998Z",
"name" : "Jordan Shoe",
"size" : "10.5",
"price" : 150.0
},
{
"id" : "2019-08-22T04:36:37.836Z",
"name" : "Nike Tee",
"size" : "M",
"price" : 35.0
}
],
"name" : "InStock",
"balance" : 7760.0
}
It's Working Fine for you.
db.collection(collectionName).updateOne({"id": "MTg4MDU3ODM2MDk2NDU0NjU3"},
{ $pull: { portfolio: { $elemMatch: { id:"2019-08-22T04:36:37.836Z"} }},

How to filter dimensional array in React

I have a dimensional array like this
{
"items": [
{
"id": "file",
"value": "File",
"childs": [
{
"value": "New",
"status": 1
},
{
"value": "New",
"status": 2
},
{
"value": "New",
"status": 1
}
]
},
{
"id": "file",
"value": "File",
"childs": [
{
"value": "New",
"status": 1
},
{
"value": "New",
"status": 2
},
{
"value": "New",
"status": 1
}
]
}
]
}
How to filter items and own childs by childs.status
You have to use filter method in order to filter your array of items based on specific test, in your case when status 3 has been found, Here is a working example:
var json = {
"items": [
{
"id": "file",
"value": "File",
"childs": [
{
"value": "New",
"status": 1
},
{
"value": "New",
"status": 2
},
{
"value": "New",
"status": 1
}
]
},
{
"id": "file",
"value": "File",
"childs": [
{
"value": "New",
"status": 1
},
{
"value": "New",
"status": 3
},
{
"value": "New",
"status": 1
}
]
}
]
};
var result = json.items.filter(function(item) {
item.childs = item.childs.filter( function (child) {
return child.status === 3;
});
return item.childs.length > 0;
});
console.log({items: result});
I've setup a repl in which i've drawn out the method to filter your json. You can find it here; feel free to play around with it.
#younel's answer mutates the original array, I've also updated my repl with his approach to show how the original json is mutated.
Essence
look through each item in items
if a child in item matches the filter criteria, then keep the item and the filtered children

Resources