Parsing Nested JSON and Manipulating It in Ruby - arrays

This is my first attempt at parsing nested JSON with Ruby. I need to go through the JSON to pull out specific values for "_id", "name", and "type" for instance. I then need to create a reference table so that I can refer to each "_id" and associated information. I also need to combine information from multiple JSON responses. I've been able to get basic information and have tried a few things I've found online. I just need a little assistance with a starting point. If anyone has any ideas of where to start with this I'd really appreciate it.
Devices JSON response hash. Each device starts with _id.
{
"api": "1.0",
"error": null,
"id": "60b5d4c3077862123cfa4443",
"result": {
"devices": [
{
"_id": "123456787786211fd31f3dd",
"batteryPowered": true,
"category": "door_lock",
"deviceTypeId": "144_1_1",
"firmware": [
{
"id": "us.144.1_1.0",
"version": "2.6"
}
],
"gatewayId": "1234567807786214fbc6bd4e",
"info": {
"firmware.stack": "3.28",
"hardware": "0",
"manufacturer": "Kwikset",
"model": "912",
"protocol": "zwave",
"zwave.node": "2",
"zwave.smartstart": "no"
},
"name": "Garage Door",
"parentDeviceId": "",
"persistent": false,
"reachable": false,
"ready": true,
"roomId": "1234567807786211fd31f3eb",
"security": "middle",
"status": "idle",
"subcategory": "",
"type": "doorlock"
},
{
"_id": "1234567897786211fd31f3ed",
"batteryPowered": true,
"category": "door_lock",
"deviceTypeId": "59_1_1129",
"firmware": [
{
"id": "us.59.18064.0",
"version": "3.3"
},
{
"id": "us.59.18065.1",
"version": "11.0"
}
],
"gatewayId": "1234567897786214fbc6bd4e",
"info": {
"firmware.stack": "6.3",
"hardware": "3",
"manufacturer": "Schlage",
"model": "BE469ZP",
"protocol": "zwave",
"zwave.node": "3",
"zwave.smartstart": "no"
},
"name": "Front Door",
"parentDeviceId": "",
"persistent": false,
"reachable": true,
"ready": true,
"roomId": "1234567807786211fd31f3ec",
"security": "high",
"status": "idle",
"subcategory": "",
"type": "doorlock"
},
{
"_id": "1234567897786211fd31f40a",
"batteryPowered": false,
"category": "switch",
"deviceTypeId": "57_20562_12344",
"firmware": [
{
"id": "us.57.29240.0",
"version": "5.25"
}
],
"gatewayId": "1234567807786214fbc6bd4e",
"info": {
"firmware.stack": "4.54",
"hardware": "255",
"manufacturer": "Honeywell",
"model": "ZW4103/39337",
"protocol": "zwave",
"zwave.node": "4",
"zwave.smartstart": "no"
},
"name": "Lamp Switch",
"parentDeviceId": "",
"persistent": false,
"reachable": true,
"ready": true,
"roomId": "1234567807786211fd31f416",
"security": "no",
"status": "idle",
"subcategory": "interior_plugin",
"type": "switch.outlet"
},
{
"_id": "1234567b07786211fd31f40e",
"batteryPowered": false,
"category": "dimmable_light",
"deviceTypeId": "57_20548_12339",
"firmware": [
{
"id": "us.57.29747.0",
"version": "5.21"
}
],
"gatewayId": "1234567d07786214fbc6bd4e",
"info": {
"firmware.stack": "4.34",
"hardware": "255",
"manufacturer": "Honeywell",
"model": "39339/ZW3107",
"protocol": "zwave",
"zwave.node": "5",
"zwave.smartstart": "no"
},
"name": "Lamp Dimmer",
"parentDeviceId": "",
"persistent": false,
"reachable": true,
"ready": true,
"roomId": "1234567807786211fd31f416",
"security": "no",
"status": "idle",
"subcategory": "dimmable_plugged",
"type": "dimmer.outlet"
}
]
}
}
There is then also a JSON response that lists the functions for each device in the same format above. However instead of "devices"=> it is "items"=> and the beach function is the _id key again.
I'd like to combine function _id tags and descriptions with the device JSON, so I can create a way to send my script "unlock door lock 1" and it subs the number with the _id of the device and the function _id.

You can start with a very rough navigator function like this:
def find_device(data, name, index)
# Filter through the device list...
data['result']['devices'].select do |device|
# ...for matching names.
device.name == name
end[index] # Take indexed entry
end
Where now you can do find_device(data, 'door_lock', 0) to dig up that entry.
Converting "door lock 1" to [ 'door_lock', 0 ] should be pretty trivial:
def to_location(str)
# Split off the name component(s) and index number
*name, index = str.split(/\s+/)
# Reassemble with underscores and -1 to account for 0-index
[ name.join('_'), index.to_i - 1 ]
end

Related

Azure Search - Cannot merge (with skill) data obtained from the KeyPhraseExtractionSkill

I am creating an indexer that takes a document, runs the KeyPhraseExtractionSkill and outputs it back to the index.
For many documents, this works out of the box. But for those records which are over 50,000, this does not work. OK, no problem; this is clearly stated in the docs.
What the docs suggest is so use the Text Split Skill. What I've done is use the Text Split skill, split the original document into pages, pass all pages to the KeyPhraseExtractionSkill. Then we need to merge them back, as we'd end up with an array of arrays of strings. Unfortunately, it seems that the Merge Skill does not accept an array of arrays, just an array.
https://i.imgur.com/dBD4qgb.png <- Link to the skillset hierarchy.
This is the error reported by Azure:
Required skill input was not of the expected type 'StringCollection'. Name: 'itemsToInsert', Source: '/document/content/pages/*/keyPhrases'. Expression language parsing issues:
What I want to achieve in the end of the day is to run the KeyPhraseExtractionSkill for text which is larger than 50,000 to add it back to the index eventually.
JSON for skillset
"#odata.context": "https://-----------.search.windows.net/$metadata#skillsets/$entity",
"#odata.etag": "\"0x8D957466A2C1E47\"",
"name": "devalbertcollectionfilesskillset2",
"description": null,
"skills": [
{
"#odata.type": "#Microsoft.Skills.Text.SplitSkill",
"name": "SplitSkill",
"description": null,
"context": "/document/content",
"defaultLanguageCode": "en",
"textSplitMode": "pages",
"maximumPageLength": 1000,
"inputs": [
{
"name": "text",
"source": "/document/content"
}
],
"outputs": [
{
"name": "textItems",
"targetName": "pages"
}
]
},
{
"#odata.type": "#Microsoft.Skills.Text.EntityRecognitionSkill",
"name": "EntityRecognitionSkill",
"description": null,
"context": "/document/content/pages/*",
"categories": [
"person",
"quantity",
"organization",
"url",
"email",
"location",
"datetime"
],
"defaultLanguageCode": "en",
"minimumPrecision": null,
"includeTypelessEntities": null,
"inputs": [
{
"name": "text",
"source": "/document/content/pages/*"
}
],
"outputs": [
{
"name": "persons",
"targetName": "people"
},
{
"name": "organizations",
"targetName": "organizations"
},
{
"name": "entities",
"targetName": "entities"
},
{
"name": "locations",
"targetName": "locations"
}
]
},
{
"#odata.type": "#Microsoft.Skills.Text.KeyPhraseExtractionSkill",
"name": "KeyPhraseExtractionSkill",
"description": null,
"context": "/document/content/pages/*",
"defaultLanguageCode": "en",
"maxKeyPhraseCount": null,
"modelVersion": null,
"inputs": [
{
"name": "text",
"source": "/document/content/pages/*"
}
],
"outputs": [
{
"name": "keyPhrases",
"targetName": "keyPhrases"
}
]
},
{
"#odata.type": "#Microsoft.Skills.Text.MergeSkill",
"name": "Merge Skill - keyPhrases",
"description": null,
"context": "/document",
"insertPreTag": " ",
"insertPostTag": " ",
"inputs": [
{
"name": "itemsToInsert",
"source": "/document/content/pages/*/keyPhrases"
}
],
"outputs": [
{
"name": "mergedText",
"targetName": "keyPhrases"
}
]
}
],
"cognitiveServices": {
"#odata.type": "#Microsoft.Azure.Search.CognitiveServicesByKey",
"key": "------",
"description": "/subscriptions/13abe1c6-d700-4f8f-916a-8d3bc17bb41e/resourceGroups/mde-dev-rg/providers/Microsoft.CognitiveServices/accounts/mde-dev-cognitive"
},
"knowledgeStore": null,
"encryptionKey": null
}```
Please let me know if there is anything else that I can add to improve the question. Thanks!
[1]: https://i.stack.imgur.com/GNf7F.png
You don't have to merge the key phrase outputs to insert them to the index.
Assuming your index already has a field called mykeyphrases of type Collection(Edm.String), to populate it with the key phrase outputs, add this indexer output field mapping:
"outputFieldMappings": [
...
{
"sourceFieldName": "/document/content/pages/*/keyPhrases/*",
"targetFieldName": "mykeyphrases"
},
...
]
The /* at the end of sourceFieldName is important to flattening the array of arrays of strings. This will also work as the skill input if you want to pass an array of strings to another skill for other enrichments.

400 Bad Request on Alexa AddOrUpdateReport using JSON from documentation

My issue is basically described in title but I'll add a little bit more details here.
I'm trying to proactively send an update report to Alexa's Event Hub for Smart Home skills, however this fails with a 400 Bad Request error, and this is what the server responds:
{
"header": {
"namespace": "System",
"name": "Exception",
"messageId": "a154410c-2364-4c5b-9028-accde5048e1e"
},
"payload": {
"code": "INVALID_REQUEST_EXCEPTION",
"description": "Event or endpoint is missing in the request."
}
}
I decided then to check if my request was wrong by sending the one that can be found on their documentation, and I'm getting the very same error.
This is the JSON copied from Amazon's documentation (I just replaced device metadata and the token):
{
"event": {
"header": {
"namespace": "Alexa.Discovery",
"name": "AddOrUpdateReport",
"payloadVersion": "3",
"messageId": "5f8a426e-01e4-4cc9-8b79-65f8bd0fd8a4"
},
"payload": {
"endpoints": [
{
"endpointId": "<unique ID of the endpoint>",
"manufacturerName": "Sample Manufacturer",
"description": "Smart Light by Sample Manufacturer",
"friendlyName": "Kitchen Light",
"additionalAttributes": {
"manufacturer" : "Sample Manufacturer",
"model" : "Sample Model",
"serialNumber": "<the serial number of the device>",
"firmwareVersion" : "<the firmware version of the device>",
"softwareVersion": "<the software version of the device>",
"customIdentifier": "<your custom identifier for the device>"
},
"displayCategories": [
"LIGHT"
],
"capabilities": [
{
"type": "AlexaInterface",
"interface": "Alexa.PowerController",
"version": "3",
"properties": {
"supported": [
{
"name": "powerState"
}
],
"proactivelyReported": true,
"retrievable": true
}
},
{
"type": "AlexaInterface",
"interface": "Alexa.BrightnessController",
"version": "3",
"properties": {
"supported": [
{
"name": "brightness"
}
],
"proactivelyReported": true,
"retrievable": true
}
}
],
"connections": [
],
"cookie": {
}
}
],
"scope": {
"type": "BearerToken",
"token": "access-token-from-Amazon"
}
}
}
}
What I don't understand is why it reports that event or endpoints are missing if they're clearly there and if this JSON is the one that they provide in their documentation.
Does anyone have this issue too?
EDIT: pasting my payload as requested.
{
"event": {
"header": {
"namespace": "Alexa.Discovery",
"name": "AddOrUpdateReport",
"payloadVersion": "3",
"messageId": "132137185729061389"
},
"payload": {
"endpoints": [
{
"endpointId": "Rmxvb2RTZW5zb3JfMDE=",
"friendlyName": "Flood sensor",
"description": "Flood sensor",
"manufacturerName": "MyCompany",
"displayCategories": [
"ACTIVITY_TRIGGER"
],
"cookie": {
"DeviceType": "FloodSensor"
},
"capabilities": [{
"interface": "Alexa",
"type": "AlexaInterface",
"version": "3"
},
{
"interface": "Alexa.EndpointHealth",
"type": "AlexaInterface",
"version": "3",
"properties": {
"supported": [{
"name": "connectivity"
}],
"proactivelyReported": true,
"retrievable": true
}
}
]
}
],
"scope": {
"type": "BearerToken",
"token": "<redacted>"
}
}
}
}

How to convert Json Array to Avro Schema

Hey i am new to Avro Schema space, needed to convert Jason Array into Avro Schema.
Below Jason is kind of client which serviceName along-with enabler-
If Enabler is true means that particular service is taken by client
If Enabler is false means that particular service is not taken by client.
{
"clientName": "Haven",
"serviceDetailsList": [
{
"serviceName": "Service1",
"enabled": true
},
{
"serviceName": "Service2",
"enabled": true
},
{
"serviceName": "Service3",
"enabled": true
},
{
"serviceName": "Service4",
"enabled": false
},
{
"serviceName": "Service5",
"enabled": false
},
{
"serviceName": "Service6",
"enabled": true
}
]
}
I worked with below schema but not getting proper response.
"fields":[
{"name": "serviceName", "type": [ "Boolean", "false" ] , "aliases":[
"service1" ]
},
{"name": "serviceName", "type": [ "Boolean", "false" ] , "aliases":[
"service2" ]
}
]
Any help would be appreciated.
Thank you all of you,again i tried and able to get correct scheam. Correct Avro Schema is-
{
"name": "modelData",
"type": "record",
"namespace": "com.hi.model",
"fields": [
{
"name": "clientName",
"type": "string"
},
{
"name": "serviceDetailsList",
"type": {
"type": "array",
"items": {
"name": "serviceDetailsList_record",
"type": "record",
"fields": [
{
"name": "serviceName",
"type": "string"
},
{
"name": "enabled",
"type": "boolean"
}
]
}
}
}
]
}

Parse this Json array and extract the value of text

Here is the full json that I want to parse and extract the text that has the value: The product is in second line 3rd row.
Can someone help?
"activities": [
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:56.8251462Z",
"serviceUrl": "XXXXXXXXXXXXXXXXXXXXXXXXX",
"channelId": "directline",
"from": {
"id": "user1"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
},
"text": "the milk"
},
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:57.6856172Z",
"localTimestamp": "2019-07-01T15:18:57.5099359+00:00",
"channelId": "directline",
"from": {
"id": "XXXXXX",
"name": "XXXXXX"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXX"
},
"text": "The product is in second line 3rd row",
"attachments": [],
"entities": [],
"replyToId": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
}
],
"watermark": "1"
}
You have specified that this is a full json you have placed in the question, But it is not full json you are missing something.
"activities":
[ //json array start from here
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:56.8251462Z",
"serviceUrl": "XXXXXXXXXXXXXXXXXXXXXXXXX",
"channelId": "directline",
"from": {
"id": "user1"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
},
"text": "the milk"
},
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:57.6856172Z",
"localTimestamp": "2019-07-01T15:18:57.5099359+00:00",
"channelId": "directline",
"from": {
"id": "XXXXXX",
"name": "XXXXXX"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXX"
},
"text": "The product is in second line 3rd row",
"attachments": [],
"entities": [],
"replyToId": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
}
], //json array ends here
these are extra lines
***"watermark": "1"
}***
if you want to use this array you need to add a '{' in very top line before
"activities": [
I am placing here a valid json now you can check it on Json parser used to parse json in objects
*Valid Json Here : *
{
"activities": [
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:56.8251462Z",
"serviceUrl": "XXXXXXXXXXXXXXXXXXXXXXXXX",
"channelId": "directline",
"from": {
"id": "user1"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
},
"text": "the milk"
},
{
"type": "message",
"id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"timestamp": "2019-07-01T15:18:57.6856172Z",
"localTimestamp": "2019-07-01T15:18:57.5099359+00:00",
"channelId": "directline",
"from": {
"id": "XXXXXX",
"name": "XXXXXX"
},
"conversation": {
"id": "XXXXXXXXXXXXXXXXXX"
},
"text": "The product is in second line 3rd row",
"attachments": [],
"entities": [],
"replyToId": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
}
],
"watermark": "1"
}

Join 2 arrays by key and value (AngularJS)

I have 2 objects
{
"_id": "58b7f36b3354c24630f6f3b0",
"name": "refcode",
"caption": "Reference",
"type": "string",
"search": false,
"required": false,
"table": true,
"expansion": true
},
and
{
"_id": "58b7f36b3354c24630f6f3c8",
"vacancyid": "0",
"refcode": "THIS IS MY REF",
"position": "Test",
"jobtype": "Temp",
"department": "Industrial",
"branch": "Office",
"startdate": "02/12/2013",
"contactname": "Person Name",
"contactemail": "person#domain",
"Q_V_TYP": "Daily",
"score": 0
},
Object one defines what a field should be and what it is called
The second object is a job description.
What i need is to match a field to each key (this even sounds confusing i my head, so here is an example)
{
"_id": "58b7f36b3354c24630f6f3c8",
"vacancyid": "0",
"refcode": {
"_id": "58b7f36b3354c24630f6f3b0",
"name": "refcode",
"caption": "Reference",
"type": "string",
"search": false,
"required": false,
"table": true,
"expansion": true,
"value": "THIS IS MY REF"
}
},
"position": "Test",
"jobtype": "Temp",
"department": "Industrial",
"branch": "Office",
"startdate": "02/12/2013",
"contactname": "Person Name",
"contactemail": "person#domain",
"Q_V_TYP": "Daily",
"score": 0
},
Here you go:
var def = {
"_id": "58b7f36b3354c24630f6f3b0",
"name": "refcode",
"caption": "Reference",
"type": "string",
"search": false,
"required": false,
"table": true,
"expansion": true
};
var jobDesc = {
"_id": "58b7f36b3354c24630f6f3c8",
"vacancyid": "0",
"refcode": "THIS IS MY REF",
"position": "Test",
"jobtype": "Temp",
"department": "Industrial",
"branch": "Office",
"startdate": "02/12/2013",
"contactname": "Person Name",
"contactemail": "person#domain",
"Q_V_TYP": "Daily",
"score": 0
};
var jobDescKeysArr = Object.keys(jobDesc);
if (jobDescKeysArr.indexOf(def.name) !== -1) {
// A match.
def.value = jobDesc[def.name];
jobDesc[def.name] = Object.assign({}, def);
console.log(jobDesc)
}

Resources