Merge two arrays and overwrite values | Dataweave - arrays

I have situation to merge global array with another custom array or if the name is the same use custom values.
Global:
{
"connections": [
{
"name": "Test SFTP",
"type": "SFTP",
"user": "sftpuser",
"password": "password",
"server": "127.0.0.1",
"port": 22,
},
{
"name": "Test FTP",
"type": "FTP",
"user": "ftpuser",
"password": "password",
"server": "127.0.0.1",
"port": 21,
}
]
}
Custom:
{
"connections": [
{
"name": "Test SFTP",
"user": "sftpuser1",
"password": "password1",
"server": "127.0.0.2",
},
{
"name": "Test FTPS",
"type": "FTPS",
"user": "ftpsuser",
"password": "password",
"server": "127.0.0.1",
"port": 990,
}
]
}
Expected:
{
"connections": [
{
"name": "Test SFTP",
"type": "SFTP",
"user": "sftpuser1",
"password": "password1",
"server": "127.0.0.2",
"port": 22,
},
{
"name": "Test FTP",
"type": "FTP",
"user": "ftpuser",
"password": "password",
"server": "127.0.0.1",
"port": 21,
},
{
"name": "Test FTPS",
"type": "FTPS",
"user": "ftpsuser",
"password": "password",
"server": "127.0.0.1",
"port": 990,
}
]
}
Global will always have all fields but custom can have name + just one field to override global.
Later on I will validate if the json is ok but for now i just need to merge and overwrite.
Thanks,
Ivan

Below script will help you.
%dw 2.0
output application/json
import mergeWith from dw::core::Objects
import * from dw::core::Arrays
var global = {
"connections": [
{
"name": "Test SFTP",
"type": "SFTP",
"user": "sftpuser",
"password": "password",
"server": "127.0.0.1",
"port": 22,
},
{
"name": "Test FTP",
"type": "FTP",
"user": "ftpuser",
"password": "password",
"server": "127.0.0.1",
"port": 21,
}
]
}
var custom = {
"connections": [
{
"name": "Test SFTP",
"user": "sftpuser1",
"password": "password1",
"server": "127.0.0.2",
},
{
"name": "Test FTPS",
"type": "FTPS",
"user": "ftpsuser",
"password": "password",
"server": "127.0.0.1",
"port": 990,
}
]
}
---
outerJoin(global.connections, custom.connections, (g) -> g.name, (c) -> c.name) map ($.l mergeWith $.r )

Related

In Avro schema, how to wrap the array to an object and put it as a union instead of putting just array as union

I have the following data and I want to wrap the OPTIONAL array as an object and then put in as a union but since I am new to this, I am not sure how to do this.
This is how I have done so far so could someone help me correct the below structure in the expected output. Note that this dataRefs is an optional field and this entire structure may or may not be present.
{
"name": "dataRefs",
"default": null,
"type": ["null",
{
"type": "array",
"items": {
"name": "dataRef",
"type": "record",
"fields": [
{
"name": "dataId",
"type": "string",
"avro.java.string": "String"
},
{
"name": "email",
"type": ["null" ,"string"],
"avro.java.string": "String"
},
{
"name": "phone",
"type": ["null" ,"string"],
"avro.java.string": "String"
},
{
"name": "userName",
"type": ["null" ,"string"],
"avro.java.string": "String"
},
{
"name": "addressRef",
"default": null,
"type": ["null", {
"name": "addressRefRecord",
"type": "record",
"fields": [
{
"name": "addrRefId",
"type": ["null","string"],
"avro.java.string": "String"
},
{
"name": "addrType",
"type": ["null","string"],
"avro.java.string": "String"
},
{
"name": "addressLine1",
"type": ["null","string"],
"avro.java.string": "String"
},
{
"name": "addressLine2",
"type": ["null","string"],
"avro.java.string": "String"
},
{
"name": "city",
"type": ["null","string"],
"avro.java.string": "String"
},
{
"name": "province",
"type": ["null","string"],
"avro.java.string": "String"
},
{
"name": "country",
"type": ["null","string"],
"avro.java.string": "String"
},
{
"name": "postalCode",
"type": ["null","string"],
"avro.java.string": "String"
}
]
}
]
}
]
}
}
]
}
My JSON data that I intend to map to above schema looks like follows:
"dataRefs": [{
"addressRef": {
"addrRefId": "0",
"addrType": "ADDRESS",
"addressLine1": "DA 81",
"addressLine2": "",
"city": "Amsterdam",
"country": "Netherlands",
"postalCode": "xxxx LN",
"province": ""
},
"dataId": "0",
"email": "xyz#abc.com"
}],
This is how I was able to do so:
{
"name": "dataRefs",
"type": [
"null",
{
"type": "record",
"name": "dataRefsObject",
"fields": [
{
"name": "dataRefsArray",
"type": {
"type": "array",
"items": {
"name": "dataRef",
"type": "record",
"fields": [
{
"name": "dataId",
"type": ["null", "string"],
"avro.java.string": "String"
},
{
"name": "userName",
"type": [
"null",
"string"
],
"avro.java.string": "String"
},
....

Invalid response from webhook: Failed to translate JSON to ExecuteHttpResponse

I have a webhook from my google assistant new actions builder platform. Webhook code is written in the dotnet vs2019. I have followed the webhook request and response format and sample jsons provided in the below links.
https://developers.google.com/assistant/conversational/reference/rest/v1/TopLevel/fulfill#User
https://developers.google.com/assistant/conversational/webhooks#request-json_1
But when I test assistant action, it is failing. Below is request and response json from logs in the google assistant test simulator.Error message just says"Invalid response from webhook: Failed to translate JSON to ExecuteHttpResponse." I don't know why it is not valid. Can someone help me resolving this.
{
"requestJson": {
"handler": {
"name": "agentcube"
},
"intent": {
"name": "",
"params": {
"phone": {
"original": "1234562869",
"resolved": 1234562869
}
},
"query": "1234562869"
},
"scene": {
"name": "Start",
"slotFillingStatus": "FINAL",
"slots": {
"Zip": {
"mode": "REQUIRED",
"status": "SLOT_UNSPECIFIED",
"updated": false,
"value": 37122
},
"Phone": {
"mode": "REQUIRED",
"status": "SLOT_UNSPECIFIED",
"updated": true,
"value": 1234562869
},
"FName": {
"mode": "REQUIRED",
"status": "SLOT_UNSPECIFIED",
"updated": false,
"value": "john doe"
}
},
"next": {
"name": "actions.scene.END_CONVERSATION"
}
},
"session": {
"id": "ABwppHFkFuGBF-UawPmIkxWpkfM1Hb5An7h8KnjR302zukmBoKA1NqDp7DfePGzYsyxT5oy--wg5Jkjj",
"params": {
"Phone": 1234562869,
"FName": "john doe",
"Zip": 37122
},
"typeOverrides": [],
"languageCode": ""
},
"user": {
"locale": "en-US",
"params": {},
"accountLinkingStatus": "NOT_LINKED",
"verificationStatus": "VERIFIED",
"packageEntitlements": [],
"gaiamint": "",
"permissions": [],
"lastSeenTime": "2021-01-28T19:21:28Z"
},
"home": {
"params": {}
},
"device": {
"capabilities": [
"SPEECH",
"RICH_RESPONSE",
"LONG_FORM_AUDIO"
]
}
}
}
Invalid response from webhook: Failed to translate JSON to ExecuteHttpResponse..
{
"responseJson": {
"Session": {
"Id": "ABwppHFkFuGBF-UawPmIkxWpkfM1Hb5An7h8KnjR302zukmBoKA1NqDp7DfePGzYsyxT5oy--wg5Jkjj",
"Params": {
"Phone": 1234562869,
"Zip": 37122,
"FName": "john doe"
}
},
"Prompt": {
"Override": false,
"FirstSimple": {
"Speech": "My webhook response",
"Text": "My response from webhook"
}
},
"Scene": {
"Name": "Start",
"SlotFillingStatus": "FINAL",
"Slots": {
"Phone": {
"Mode": "REQUIRED",
"Status": "SLOT_UNSPECIFIED",
"Updated": true,
"value": 1234562869
},
"FName": {
"Mode": "REQUIRED",
"Status": "SLOT_UNSPECIFIED",
"Updated": false,
"value": "john doe"
},
"Zip": {
"Mode": "REQUIRED",
"Status": "SLOT_UNSPECIFIED",
"Updated": false,
"value": 37122
}
},
"Next": {
"Name": "actions.scene.END_CONVERSATION"
}
}
}
}
Make sure the response that you code is returning is json by checking with a linter, which this seems to checks (i.e. everything inside the ResponseJson {}
Please see my response to a similar issue at https://stackoverflow.com/a/66512370/10537202 to use the webhook playground.

retrieving array from json as a table in Kusto

I am trying to retrieve all the rows from a Json array. The json is similar to the one shown below.
{
"messageId": "123",
"fileName": "abc.json",
"payload": {
"routeStatus": "FINAL",
"activities": [
{
"durationSeconds": 1800,
"location": {
"longitude": 151.2603,
"latitude": -33.7644
},
"type": "DEPART",
"slot": {
"start": "2020-04-14T19:05:00.0000000Z",
"cost": null,
"end": "2020-04-15T03:30:00.0000000Z"
}
},
{
"durationSeconds": 1100,
"type": "DRIVE"
},
{
"durationSeconds": 360,
"location": {
"longitude": 151.21814,
"latitude": -33.756319
},
"type": "SERVICE",
"slot": {
"start": "2020-04-14T20:00:00.0000000Z",
"cost": null,
"end": "2020-04-15T00:45:00.0000000Z"
}
},
{
"durationSeconds": 164,
"type": "DRIVE"
}
],
"truck": "XYZ"
}
}
I would like to get all the attributes under the activities in a table as I would need to filter and join to other tables. I am only able to retrieve one row from the array. Any pointers would be helpful.
you can use mv-expand or mv-apply.
for example:
print d = dynamic({
"messageId": "123",
"fileName": "abc.json",
"payload": {
"routeStatus": "FINAL",
"activities": [
{
"durationSeconds": 1800,
"location": {
"longitude": 151.2603,
"latitude": -33.7644
},
"type": "DEPART",
"slot": {
"start": "2020-04-14T19:05:00.0000000Z",
"cost": null,
"end": "2020-04-15T03:30:00.0000000Z"
}
},
{
"durationSeconds": 1100,
"type": "DRIVE"
},
{
"durationSeconds": 360,
"location": {
"longitude": 151.21814,
"latitude": -33.756319
},
"type": "SERVICE",
"slot": {
"start": "2020-04-14T20:00:00.0000000Z",
"cost": null,
"end": "2020-04-15T00:45:00.0000000Z"
}
},
{
"durationSeconds": 164,
"type": "DRIVE"
}
],
"truck": "XYZ"
}
})
| mv-expand d.payload.activities
| project durationSeconds = tolong(d_payload_activities.durationSeconds), type = tostring(d_payload_activities.type)

Parse this Json array and extract the value of text

Here is the full json that I want to parse and extract the text that has the value: The product is in second line 3rd row.
Can someone help?
"activities": [
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:56.8251462Z",
"serviceUrl": "XXXXXXXXXXXXXXXXXXXXXXXXX",
"channelId": "directline",
"from": {
"id": "user1"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
},
"text": "the milk"
},
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:57.6856172Z",
"localTimestamp": "2019-07-01T15:18:57.5099359+00:00",
"channelId": "directline",
"from": {
"id": "XXXXXX",
"name": "XXXXXX"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXX"
},
"text": "The product is in second line 3rd row",
"attachments": [],
"entities": [],
"replyToId": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
}
],
"watermark": "1"
}
You have specified that this is a full json you have placed in the question, But it is not full json you are missing something.
"activities":
[ //json array start from here
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:56.8251462Z",
"serviceUrl": "XXXXXXXXXXXXXXXXXXXXXXXXX",
"channelId": "directline",
"from": {
"id": "user1"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
},
"text": "the milk"
},
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:57.6856172Z",
"localTimestamp": "2019-07-01T15:18:57.5099359+00:00",
"channelId": "directline",
"from": {
"id": "XXXXXX",
"name": "XXXXXX"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXX"
},
"text": "The product is in second line 3rd row",
"attachments": [],
"entities": [],
"replyToId": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
}
], //json array ends here
these are extra lines
***"watermark": "1"
}***
if you want to use this array you need to add a '{' in very top line before
"activities": [
I am placing here a valid json now you can check it on Json parser used to parse json in objects
*Valid Json Here : *
{
"activities": [
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:56.8251462Z",
"serviceUrl": "XXXXXXXXXXXXXXXXXXXXXXXXX",
"channelId": "directline",
"from": {
"id": "user1"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
},
"text": "the milk"
},
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:57.6856172Z",
"localTimestamp": "2019-07-01T15:18:57.5099359+00:00",
"channelId": "directline",
"from": {
"id": "XXXXXX",
"name": "XXXXXX"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXX"
},
"text": "The product is in second line 3rd row",
"attachments": [],
"entities": [],
"replyToId": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
}
],
"watermark": "1"
}

Json schema for array of objects doesn't validate

I have this schema for a json response
{
"title": "Products",
"description": "schema for products",
"type": "array",
"properties": {
"id": {
"description": "id of a product",
"type": "integer"
},
"name": {
"description": "name of the product",
"type": "string"
},
"created_at": {
"description": "record created_at",
"type": "string",
"format": "date-time"
},
"updated_at": {
"description": "record updated_at",
"type": "string",
"format": "date-time"
}
},
"required": ["id", "name"]
}
and I want to match this schema with this json
[{
"id": 1,
"name": "Cricket Ball"
}, {
"id": 2,
"name": "Soccer Ball"
}, {
"id": 3,
"name": "football ball"
}, {
"id": 4,
"name": "Basketball ball"
}, {
"id": 5,
"name": "Table Tennis ball"
}, {
"id": 6,
"name": "Tennis ball"
}]
This schema matches the response but it also matches the schema in which the required field is this
"required": ["ids", "names"]
I think the schema is validated against the array and the objects in the array are not validated.
The way you have it set up now, your properties key refers to the array itself, not to each item, and is being ignored (because arrays don't have properties, they just have items). You need to use the items key to validate each item in the array, like so:
{
"title": "Products",
"description": "schema for products",
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"description": "id of a product",
"type": "integer"
},
"name": {
"description": "name of the product",
"type": "string"
},
"created_at": {
"description": "record created_at",
"type": "string",
"format": "date-time"
},
"updated_at": {
"description": "record updated_at",
"type": "string",
"format": "date-time"
}
},
"required": ["id", "name"]
}
}
try map
new_array = response.map{ |k| { 'id': k['properties']['id']['description'], 'name': k['properties']['name']['description'] } }

Resources