Related
After spending many hours of reading the documentation, following some tutorials and trial & error, i just can't figure it out; how can I transform the following complex object with key objects to an array using a data flow in Azure Data Factory?
Input
{
"headers": {
"Content-Length": 1234
},
"body": {
"00b50a39-8591-3db3-88f7-635e2ec5c65a": {
"id": "00b50a39-8591-3db3-88f7-635e2ec5c65a",
"name": "Example 1",
"date": "2023-02-09"
},
"0c206312-2348-391b-99f0-261323a94d95": {
"id": "0c206312-2348-391b-99f0-261323a94d95",
"name": "Example 2",
"date": "2023-02-09"
},
"0c82d1e4-a897-32f2-88db-6830a21b0a43": {
"id": "00b50a39-8591-3db3-88f7-635e2ec5c65a",
"name": "Example 3",
"date": "2023-02-09"
},
}
}
Expected output
[
{
"id": "00b50a39-8591-3db3-88f7-635e2ec5c65a",
"name": "Example 1",
"date": "2023-02-09"
},
{
"id": "0c206312-2348-391b-99f0-261323a94d95",
"name": "Example 2",
"date": "2023-02-09"
},
{
"id": "00b50a39-8591-3db3-88f7-635e2ec5c65a",
"name": "Example 3",
"date": "2023-02-09"
}
]
AFAIK, Your JSON keys are dynamic. So, getting the desired result using dataflow might not be possible.
In this case, you can try the below approach as a workaround. This will work only if all of your key's length is same.
This is my Pipeline:
First I have used a lookup activity to get the JSON file and converted the lookup output to a string and stored in a variable using below expression.
#substring(string(activity('Lookup1').output.value[0].body),2,sub(length(string(activity('Lookup1').output.value[0].body)),4)).
Then I have used split on that String variable with '},"' and stored in an array variable using below expression.
#split(variables('res_str'),'},"')
It will give the array like below.
Give that array to a ForEach and inside ForEach use an append variable activity to store the keys into an array with below expression.
#take(item(), 36)
Now, I got the list of keys in an array, after the above ForEach use another ForEach activity to get the desired array of objects. Use append variable actvity inside ForEach and give the below expression for it.
#activity('Lookup1').output.value[0].body[item()]
Result array after ForEach will be:
If you want to store the above JSON into a file, you need to use OPENJSON from SQL. This is because copy activity additonal column only supports string type not an array type.
Use a SQL dataset on copy activity source and give the below SQL script in the query.
DECLARE #json NVARCHAR(MAX)
SET #json =
N'#{variables('json_arr')}'
SELECT * FROM
OPENJSON ( #json )
WITH (
id varchar(200) '$.id' ,
name varchar(32) '$.name',
date varchar(32) '$.date'
)
In Sink, give a JSON dataset and select Array of Objects as File pattern.
Execute the pipeline and you will get the above array inside a file.
This is my Pipeline JSON:
{
"name": "pipeline1",
"properties": {
"activities": [
{
"name": "Lookup1",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobFSReadSettings",
"recursive": true,
"enablePartitionDiscovery": false
},
"formatSettings": {
"type": "JsonReadSettings"
}
},
"dataset": {
"referenceName": "Json1",
"type": "DatasetReference"
},
"firstRowOnly": false
}
},
{
"name": "Lookup output to Str",
"description": "",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Lookup1",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "res_str",
"value": {
"value": "#substring(string(activity('Lookup1').output.value[0].body),2,sub(length(string(activity('Lookup1').output.value[0].body)),4))",
"type": "Expression"
}
}
},
{
"name": "Split Str to array",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Lookup output to Str",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "split_arr",
"value": {
"value": "#split(variables('res_str'),'},\"')",
"type": "Expression"
}
}
},
{
"name": "build keys array using split array",
"type": "ForEach",
"dependsOn": [
{
"activity": "Split Str to array",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "#variables('split_arr')",
"type": "Expression"
},
"isSequential": true,
"activities": [
{
"name": "take first 36 chars of every item",
"type": "AppendVariable",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"variableName": "keys_array",
"value": {
"value": "#take(item(), 36)",
"type": "Expression"
}
}
}
]
}
},
{
"name": "build final array using keys array",
"type": "ForEach",
"dependsOn": [
{
"activity": "build keys array using split array",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "#variables('keys_array')",
"type": "Expression"
},
"isSequential": true,
"activities": [
{
"name": "Append variable1",
"description": "append every object to array",
"type": "AppendVariable",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"variableName": "json_arr",
"value": {
"value": "#activity('Lookup1').output.value[0].body[item()]",
"type": "Expression"
}
}
}
]
}
},
{
"name": "Just for Res show",
"type": "SetVariable",
"dependsOn": [
{
"activity": "build final array using keys array",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "final_res_show",
"value": {
"value": "#variables('json_arr')",
"type": "Expression"
}
}
},
{
"name": "Copy data1",
"type": "Copy",
"dependsOn": [
{
"activity": "Just for Res show",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": "DECLARE #json NVARCHAR(MAX)\nSET #json = \n N'#{variables('json_arr')}' \n \nSELECT * FROM \n OPENJSON ( #json ) \nWITH ( \n id varchar(200) '$.id' , \n name varchar(32) '$.name', \n date varchar(32) '$.date'\n )",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "JsonWriteSettings",
"filePattern": "arrayOfObjects"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "AzureSqlTable1",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "Target_JSON",
"type": "DatasetReference"
}
]
}
],
"variables": {
"res_str": {
"type": "String"
},
"split_arr": {
"type": "Array"
},
"keys_array": {
"type": "Array"
},
"final_res_show": {
"type": "Array"
},
"json_arr": {
"type": "Array"
}
},
"annotations": []
}
}
Result file:
I have Following 2 Json in 2 rows.
{
"attributes": [{
"name": "text-1580797977710",
"value": "Nikesh Niroula"
}, {
"name": "email-1580797979166",
"value": "nikesh#gmail.com"
}]
}
{
"attributes": [{
"name": "text-1580797977720",
"value": "Denver"
}, {
"name": "text-1580797977723",
"value": "colarado"
},
{
"name": "text-1580797977727",
"value": "USA"
}
]
}
I need the above json to be aggregated into one single array by using postgresql, the expected result would be as below. I tried using json_agg but this will add a inner array inside the main array. There might be multiple json and not only 2.
{
"attributes": [{
"name": "text-1580797977710",
"value": "Nikesh Niroula"
}, {
"name": "email-1580797979166",
"value": "nikesh#gmail.com"
}, {
"name": "text-1580797977720",
"value": "Denver"
}, {
"name": "text-1580797977723",
"value": "colarado"
},
{
"name": "text-1580797977727",
"value": "USA"
}
]
}
You need to unnest them before you feed them to the agg function. If the table is "j" and the column is "x", then:
select jsonb_build_object('attributes',jsonb_agg(value))
from j, jsonb_array_elements(x->'attributes');
You can use
the concatenation operator (||) along with jsonb_array_elements jsonb_agg functions to solve your problem.
with t (id,
attr1,
attr2) as (
values (1,
'{
"attributes": [
{
"name": "one-1",
"value": "Nikesh Niroula"
}]}'::jsonb,
'{"attributes": [
{
"name": "one-2",
"value": "Nikesh Niroula"
}]}'::jsonb) ,
(1,
'{
"attributes": [
{
"name": "two-1",
"value": "Nikesh Niroula"
}]}'::jsonb,
'{"attributes": [
{
"name": "two-2",
"value": "Nikesh Niroula"
}]}'::jsonb) ,
(1,
'{
"attributes": [
{
"name": "three-1",
"value": "Nikesh Niroula"
}]}'::jsonb,
'{"attributes": [
{
"name": "three-2",
"value": "Nikesh Niroula"
}]}'::jsonb) ,
(2,
'{
"attributes": [
{
"name": "four-1",
"value": "Nikesh Niroula"
}]}'::jsonb,
'{"attributes": [
{
"name": "four-2",
"value": "Nikesh Niroula"
}]}'::jsonb) )
select
id ,
jsonb_agg(value)
from
t,
jsonb_array_elements(attr1->'attributes' ||(attr2->'attributes'))
group by
1;
So here I have an object that I am trying to map:
var bakery = {
"items":
{
"item":[
{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters": {
"batter":[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
]
},
"topping":[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
},
...
...
...
]
}
}
This is the target outcome
var target = [{
"id": 1, //as an int
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters": "all of the batter types as a string",
"ingredients": [],//a copy of all the toppings
"countOfFillings": 0
}];
And here is my mapping function
// creates variable bakeryArray that contains the actual Array inside of Baker var
var bakeryArray = bakery.items.item
// newCakes var invoked map function with the bakeryArray
var newCakes = bakeryArray.map(mapCakes)
function mapCakes(oldCakes) {
let batter = oldCakes.batters.batter
console.log(batter, "batter Logged")
var newCakesObject = {
type: oldCakes.type,
name: oldCakes.name,
ppu: oldCakes.ppu,
batters: batter.type,
ingredients: "ingridients",
countOfFillings: "total number of ingrediensts"
};
return newCakesObject;
};
I am running into problems in getting the Batter, Ingredients, and countOfFillings from the old object into the new one.
The only thing I can think of doing in order to get the batters in the newCakesObject is that I have to create another mapping function for the batter (I put my attempt at that below)? and then invoke that in the mapCakes function under batters? but every time I create another function for that I get an error saying that it's undefined once I call newBatterArray in the console
var newBatterArray = bakeryArray.map(mapBatters)
function mapBatters(oldarray) {
let theBatters = oldarray.batters.batter
console.log(theBatters.type, "we ran")
var newBatters = {
type: theBatters.type
}
return newBatters;
}
To have a much more clear interpretation of your bakery object I have tweaked it a bit
var bakery = {
"items":[
{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
],
"toppings":[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
},
{
"id": "0002",
"type": "donut",
"name": "Cake",
"ppu": 0.65,
"batters":[
{ "id": "1001", "type": "Regular1" },
{ "id": "1002", "type": "Chocolate1" },
{ "id": "1003", "type": "Blueberry1" },
{ "id": "1004", "type": "Devil's Food1" }
],
"toppings":[
{ "id": "5001", "type": "None1" },
{ "id": "5002", "type": "Glazed1" },
{ "id": "5005", "type": "Sugar1" },
{ "id": "5007", "type": "Powdered Sugar1" },
{ "id": "5006", "type": "Chocolate with Sprinkles1" },
{ "id": "5003", "type": "Chocolate1" },
{ "id": "5004", "type": "Maple1" }
]
},
...
...
...
...
]
}
Now You can iterate through each item and build your target array as follows
var target = [];
// define reducer function for each item in bakery.items
const reduceToTarget = item => {
var obj = {};
obj.id = item.id;
obj.type = item.type;
obj.name = item.name;
obj.ppu = item.ppu;
obj.batters = '';
item.batters.forEach(b => obj.batters+=b.type+'|');
obj.ingredients = item.toppings;
target.push(obj);
}
// Now you can call the reduceToTarget function to get the desired target list/array
bakery.items.forEach(reduceToTarget);
The output for this looks something like this
target = [
{
id: "0001"
type: "donut"
name: "Cake"
ppu: 0.55
batters: "Regular|Chocolate|Blueberry|Devil's Food|",
ingredients : [/* list of ingredients*/]
},
{
id: "0002"
type: "donut"
name: "Cake"
ppu: 0.65
batters: "Regular|Chocolate|Blueberry|Devil's Food|",
ingredients : [/* list of ingredients*/]
}
]
NOTE:
For getting the countOfFillings you can simply call length() function on your ingredients list for any element in target
I am new to getting JSON into SQL Server 2016, I thought I had it down, but I notice that I am missing some details from the array, looking at the image, there are four address', but I saw there are some more address' missing for example Burrows Rd, and Urana RD. How can I make sure that all the address' captured?
https://i.stack.imgur.com/erzBV.jpg
#json nvarchar(max)
#json = N'{
"response": [
{
"application": {
"info": {
"dat_id": "010.2018.00036494.001",
"development_type": "Residential - Single new dwelling",
"application_type": "DA",
"last_modified_date": "2018-12-03T11:35:24+11:00",
"description": "Residence, Garage & Colorbond Shed, Demolition of Existing Residence & Tree Removal",
"authority": {
"ref": "http://gemini:82/ApplicationTracker/atdis/1.0",
"name": "AlburyCity"
},
"lodgement_date": "2018-10-26T00:00:00+11:00",
"determination_date": null,
"determination_type": "Pending",
"status": "In Progress",
"notification_start_date": null,
"notification_end_date": null,
"officer": "David Flood",
"estimated_cost": "Not applicable.",
"related_apps": [ ]
},
"reference": {
"more_info_url": "http://gemini:82/ApplicationTracker/Application/ApplicationDetails/010.2018.00036494.001/",
"comments_url": null
},
"locations": [
{
"land_title_ref": {
"torrens": {
"lot": "11",
"section": null,
"dpsp_id": "DP 1031272"
}
},
"address": {
"street": "680 Centaur RD",
"suburb": "HAMILTON VALLEY",
"state": "NSW",
"postcode": "2641"
},
"geometry": null
},
{
"land_title_ref": {
"torrens": {
"lot": "11",
"section": null,
"dpsp_id": "DP 1031272"
}
},
"address": {
"street": "Urana RD",
"suburb": "HAMILTON VALLEY",
"state": "NSW",
"postcode": "2641"
},
"geometry": null
},
{
"land_title_ref": {
"torrens": {
"lot": "11",
"section": null,
"dpsp_id": "DP 1031272"
}
},
"address": {
"street": "Burrows RD",
"suburb": "HAMILTON VALLEY",
"state": "NSW",
"postcode": "2641"
},
"geometry": null
}
],
"events": [
{
"id": "3680347",
"timestamp": "2018-11-01T15:58:00+11:00",
"description": "Public Notification",
"event_type": "PNOT",
"status": "COMP"
},
{
"id": "3680348",
"timestamp": "2018-11-01T15:58:00+11:00",
"description": "Referral Engineering",
"event_type": "ENG",
"status": "COMP"
},
{
"id": "3680349",
"timestamp": "2018-11-01T15:59:00+11:00",
"description": "Referal Parks & Recreation",
"event_type": "TREE",
"status": "COMP"
},
{
"id": "3680350",
"timestamp": "2018-11-01T16:00:00+11:00",
"description": "Acknowledgement to Applicant",
"event_type": "ACKN",
"status": "COMP"
},
{
"id": "3683617",
"timestamp": "2018-11-21T14:59:00+11:00",
"description": "Site Assessment Inspection",
"event_type": "SITE",
"status": "PASS"
},
{
"id": "3685155",
"timestamp": "2018-12-03T11:37:00+11:00",
"description": "Assessment Report",
"event_type": "ASS3",
"status": "COMP"
}
],
"documents": [
{
"ref": "DOC18/163129",
"title": "Statement of Environmental Effects - SEE",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=6hF7YEiv0qE=&fileName=Statement+of+Environmental+Effects+-+SEE.PDF"
},
{
"ref": "DOC18/163139",
"title": "Site Plan and Elevations",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=1Gv3GVOIHCM=&fileName=Site+Plan+and+Elevations.PDF"
}
],
"people": [
{
"name": "Karyn Ford",
"role": "Applicant",
"contact": "6023 8287"
}
],
"extended": {
"software_vendor": "Civica",
"software_vendor_url": "http://www.civica.com.au"
}
}
},
{
"application": {
"info": {
"dat_id": "017.2018.00036017.001",
"development_type": "Subdivision Only",
"application_type": "SCC",
"last_modified_date": "2018-12-03T10:19:25+11:00",
"description": "Roads, Sewer, Water & Drainage for Two (2) Lot Torrens Title Subdivisi on",
"authority": {
"ref": "http://gemini:82/ApplicationTracker/atdis/1.0",
"name": "AlburyCity"
},
"lodgement_date": "2018-11-14T00:00:00+11:00",
"determination_date": "2018-11-29T00:00:00+11:00",
"determination_type": "Approved under delegation",
"status": "Determined",
"notification_start_date": null,
"notification_end_date": null,
"officer": "Sharna Holland",
"estimated_cost": "Not applicable.",
"related_apps": [ "http://gemini:82/ApplicationTracker/atdis/1.0/010.2018.00036017.001.json" ]
},
"reference": {
"more_info_url": "http://gemini:82/ApplicationTracker/Application/ApplicationDetails/017.2018.00036017.001/",
"comments_url": null
},
"locations": [
{
"land_title_ref": {
"torrens": {
"lot": "A",
"section": null,
"dpsp_id": "DP 161410"
}
},
"address": {
"street": "419 Hovell ST",
"suburb": "SOUTH ALBURY",
"state": "NSW",
"postcode": "2640"
},
"geometry": null
},
{
"land_title_ref": {
"torrens": {
"lot": "A",
"section": null,
"dpsp_id": "DP 161410"
}
},
"address": {
"street": "Charles ST",
"suburb": "SOUTH ALBURY",
"state": "NSW",
"postcode": "2640"
},
"geometry": null
}
],
"events": [
{
"id": "3683888",
"timestamp": "2018-11-23T14:03:00+11:00",
"description": "Acknowledgement to Applicant",
"event_type": "ACKN",
"status": "COMP"
},
{
"id": "3683902",
"timestamp": "2018-11-23T15:21:00+11:00",
"description": "Referral Engineering",
"event_type": "ENG",
"status": "COMP"
},
{
"id": "3683903",
"timestamp": "2018-11-23T15:21:00+11:00",
"description": "Referral Building Surveyor 3",
"event_type": "BS3",
"status": "COMP"
},
{
"id": "3683904",
"timestamp": "2018-11-23T15:21:00+11:00",
"description": "Referral Trainee Town Planner",
"event_type": "TTP1",
"status": "COMP"
},
{
"id": "3684791",
"timestamp": "2018-11-29T14:36:00+11:00",
"description": "Collected Determination",
"event_type": "COLL",
"status": "COMP"
}
],
"documents": [
{
"ref": "DOC18/177392",
"title": "Subdivision Works Certificate",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=8qU1Pkawfvg=&fileName=Subdivision+Works+Certificate.PDF"
}
],
"people": [
{
"name": "Eslers Land Consulting",
"role": "Applicant",
"contact": "6021 1322"
}
],
"extended": {
"software_vendor": "Civica",
"software_vendor_url": "http://www.civica.com.au"
}
}
},
{
"application": {
"info": {
"dat_id": "010.2016.00034838.001",
"development_type": "Residential - New multi unit",
"application_type": "DA",
"last_modified_date": "2018-12-03T09:36:09+11:00",
"description": "Twenty (20) Detached Self Contained Residences - Kensington Gardens Retirement Village",
"authority": {
"ref": "http://gemini:82/ApplicationTracker/atdis/1.0",
"name": "AlburyCity"
},
"lodgement_date": "2016-08-17T00:00:00+10:00",
"determination_date": "2016-10-24T00:00:00+11:00",
"determination_type": "Approved under delegation",
"status": "Determined",
"notification_start_date": null,
"notification_end_date": null,
"officer": "Christopher Eldred",
"estimated_cost": "Not applicable.",
"related_apps": [ "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.001.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.002.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.003.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.004.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.005.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.006.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.007.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.008.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.009.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.010.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.011.json", "http://gemini:82/ApplicationTracker/atdis/1.0/011.2016.00034838.012.json" ]
},
"reference": {
"more_info_url": "http://gemini:82/ApplicationTracker/Application/ApplicationDetails/010.2016.00034838.001/",
"comments_url": null
},
"locations": [
{
"land_title_ref": {
"torrens": {
"lot": "2",
"section": null,
"dpsp_id": "DP 874732"
}
},
"address": {
"street": "100 Table Top RD",
"suburb": "THURGOONA",
"state": "NSW",
"postcode": "2640"
},
"geometry": null
}
],
"events": [
{
"id": "3583145",
"timestamp": "2016-08-17T14:13:00+10:00",
"description": "Further Information Requested",
"event_type": "INFO",
"status": "COMP"
},
{
"id": "3573096",
"timestamp": "2016-08-18T15:34:00+10:00",
"description": "Public Notification",
"event_type": "PNOT",
"status": "COMP"
},
{
"id": "3573097",
"timestamp": "2016-08-18T15:37:00+10:00",
"description": "Referral Building Surveyor 2",
"event_type": "BS2",
"status": "COMP"
},
{
"id": "3573098",
"timestamp": "2016-08-18T15:37:00+10:00",
"description": "Referral Plumbing Inspector",
"event_type": "PI",
"status": "COMP"
},
{
"id": "3573099",
"timestamp": "2016-08-18T15:37:00+10:00",
"description": "Referral Engineering",
"event_type": "ENG",
"status": "COMP"
},
{
"id": "3573100",
"timestamp": "2016-08-18T15:38:00+10:00",
"description": "Referral Environmental Planner",
"event_type": "ENV",
"status": "COMP"
},
{
"id": "3573103",
"timestamp": "2016-08-18T15:43:43+10:00",
"description": "Acknowledgement to Applicant",
"event_type": "ACKN",
"status": "COMP"
},
{
"id": "3575194",
"timestamp": "2016-09-06T00:00:00+10:00",
"description": "Assessment Report",
"event_type": "ASS3",
"status": "COMP"
}
],
"documents": [
{
"ref": "DOC16/209893",
"title": "Statement of Environmental Effects - SEE",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=sQkVV9rEsTU=&fileName=Statement+of+Environmental+Effects+-+SEE.PDF"
},
{
"ref": "DOC16/209896",
"title": "Site Plan & Elevations",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=3dRqEZHzGeo=&fileName=Site+Plan+%26+Elevations.PDF"
},
{
"ref": "DOC16/211819",
"title": "Assessment Report",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=DVTQkQQqbns=&fileName=Assessment+Report.PDF"
},
{
"ref": "DOC16/240764",
"title": "Development Consent",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=TY+Y3zjyDKw=&fileName=Development+Consent.PDF"
}
],
"people": [
{
"name": "Kensington Gardens Lifestyle Estates",
"role": "Applicant",
"contact": "6049 3100"
}
],
"extended": {
"software_vendor": "Civica",
"software_vendor_url": "http://www.civica.com.au"
}
}
},
{
"application": {
"info": {
"dat_id": "010.2018.00036468.001",
"development_type": "Residential - Single new dwelling",
"application_type": "DA",
"last_modified_date": "2018-11-30T17:17:25+11:00",
"description": "Residence, Garage and Retaining Walls",
"authority": {
"ref": "http://gemini:82/ApplicationTracker/atdis/1.0",
"name": "AlburyCity"
},
"lodgement_date": "2018-10-19T00:00:00+11:00",
"determination_date": "2018-11-26T00:00:00+11:00",
"determination_type": "Approved under delegation",
"status": "Determined",
"notification_start_date": null,
"notification_end_date": null,
"officer": "David Flood",
"estimated_cost": "Not applicable.",
"related_apps": [ "http://gemini:82/ApplicationTracker/atdis/1.0/011.2018.00036468.001.json" ]
},
"reference": {
"more_info_url": "http://gemini:82/ApplicationTracker/Application/ApplicationDetails/010.2018.00036468.001/",
"comments_url": null
},
"locations": [
{
"land_title_ref": {
"torrens": {
"lot": "218",
"section": null,
"dpsp_id": "DP 1228226"
}
},
"address": {
"street": "20 Stockman CRCT",
"suburb": "THURGOONA",
"state": "NSW",
"postcode": "2640"
},
"geometry": null
}
],
"events": [
{
"id": "3678966",
"timestamp": "2018-10-25T10:47:00+11:00",
"description": "Public Notification",
"event_type": "PNOT",
"status": "COMP"
},
{
"id": "3678967",
"timestamp": "2018-10-25T10:48:00+11:00",
"description": "Referral Engineering",
"event_type": "ENG",
"status": "COMP"
},
{
"id": "3678974",
"timestamp": "2018-10-25T10:51:00+11:00",
"description": "Acknowledgement to Applicant",
"event_type": "ACKN",
"status": "COMP"
},
{
"id": "3681955",
"timestamp": "2018-11-01T15:49:00+11:00",
"description": "Site Assessment Inspection",
"event_type": "SITE",
"status": "COMP"
},
{
"id": "3684251",
"timestamp": "2018-11-27T10:24:00+11:00",
"description": "Collected Determination",
"event_type": "COLL",
"status": "COMP"
}
],
"documents": [
{
"ref": "DOC18/159026",
"title": "Statement of Environmental Effects - SEE",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=MaiWkTs8V+g=&fileName=Statement+of+Environmental+Effects+-+SEE.PDF"
},
{
"ref": "DOC18/159033",
"title": "Site Plan and Elevations and Superseded Plan",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=2xpqra8gNb0=&fileName=Site+Plan+and+Elevations+and+Superseded+Plan.PDF"
},
{
"ref": "DOC18/162569",
"title": "Assessment Report - Bldg Residential",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=bKYCMP01aJE=&fileName=Assessment+Report+-+Bldg+Residential.PDF"
},
{
"ref": "DOC18/168584",
"title": "Development Consent",
"document_url": "http://gemini:82/ApplicationTracker/atdis/1.0/Document/Download?key=SxGG/yKi68s=&fileName=Development+Consent.PDF"
}
],
"people": [
{
"name": "Alatalo Bros",
"role": "Applicant",
"contact": "02 6055 0180"
}
],
"extended": {
"software_vendor": "Civica",
"software_vendor_url": "http://www.civica.com.au"
}
}
}
],
"count": 4,
"pagination": {
"previous": null,
"next": 2,
"current": 1,
"per_page": 4,
"count": 24091,
"pages": 6023
}
}'
select *
from OPENJSON(#json, '$.response')
with
([lot] varchar(200) '$.application.locations[0].land_title_ref.torrens.lot',
[section] varchar(200) '$.application.locations[0].land_title_ref.torrens.section',
[dpsp_id] varchar(200) '$.application.locations[0].land_title_ref.torrens.dpsp_id',
[Street] varchar(200) '$.application.locations[0].address.street',
[suburb] varchar(200) '$.application.locations[0].address.suburb',
[state] varchar(200) '$.application.locations[0].address.state',
[postcode] varchar(200) '$.application.locations[0].address.postcode',
[geometry] varchar(200) '$.application.locations[0].geometry'
)
Try this:
select L.*
from OPENJSON(#json,'$.response') R
CROSS APPLY OPENJSON(R.[value], '$.application.locations')
with
(
[lot] varchar(200) '$.land_title_ref.torrens.lot',
[section] varchar(200) '$.land_title_ref.torrens.section',
[dpsp_id] varchar(200) '$.land_title_ref.torrens.dpsp_id',
[Street] varchar(200) '$.address.street',
[suburb] varchar(200) '$.address.suburb',
[state] varchar(200) '$.address.state',
[postcode] varchar(200) '$.address.postcode',
[geometry] varchar(200) '$.geometry'
) L
The locations is an array, so we need to use cross apply and OPENSJON to get all elements.
I have to index a json to Elastic which look like the below format. My problem is that the key "variable" is array that contains json objects (I thought about "nested" datatype of Elastic) but some of those objects it's possible to contain nested json arrays inside them. (see variable CUSTOMERS).
POST /example_data/data {
"process_name": "TEST_PROCESS",
"process_version ": 0,
"process_id": "1111",
"activity_id": "111",
"name": "update_data",
"username": "testUser",
"datetime": "2018-01-01 10:00:00",
"variables": [{
"name": "ΒΑΝΚ",
"data_type": "STRING",
"value": "EUROBANK"
},{
"name": "CITY",
"data_type": "STRING",
"value": "LONDON"
}, {
"name": "CUSTOMERS",
"data_type": "ENTITY",
"value": [{
"variables": [{
"name": "CUSTOMER_NAME",
"data_type": "STRING",
"value": "JOHN"
}, {
"name": " CUSTOMER_CITY",
"data_type": "STRING",
"value": "LONDON"
}
]
}
]
}, {
"name": "CUSTOMERS",
"data_type": "ENTITY",
"value": [{
"variables": [{
"name": "CUSTOMER_NAME",
"data_type": "STRING",
"value": "ΑΘΗΝΑ"
}, {
"name": " CUSTOMER_CITY ",
"data_type": "STRING",
"value": "LIVERPOOL"
}, {
"name": " CUSTOMER_NUMBER",
"data_type": "STRING",
"value": "1234567890"
}
]
}
]
}
] }
When I'm trying to index it I get the following error
{ "error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "Can't merge a non object mapping [variables.value] with an object mapping [variables.value]"
}
],
"type": "illegal_argument_exception",
"reason": "Can't merge a non object mapping [variables.value] with an object mapping [variables.value]" }, "status": 400 }
Mapping
{ "example_data": {
"mappings": {
"data": {
"properties": {
"activity_id": {
"type": "text"
},
"name": {
"type": "text"
},
"process_name": {
"type": "text"
},
"process_version": {
"type": "integer"
}
"process_id": {
"type": "text"
},
"datetime": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"username": {
"type": "text",
"analyzer": "greek"
},
"variables": {
"type": "nested",
"properties": {
"data_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"value": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}}}
When I remove the variable CUSTOMERS that contains the array, then It works properly because there are only json objects.
Is there a way to handle that? Thanks in advance