OPENJSON() optimization for poorly structured API response - sql-server

I'm trying to use the TSheets API to pull data into my internal database, but the responses come back in a way that I can't figure out how to efficiently get it into table structure.
I've simplified the response for demo purposes, but it basically looks like this:
{
"results": {
"users": {
"12345": {
"id": 12345,
"first_name": "Demo",
"last_name": "User",
"username": "demo#gmail.com",
"email": "demo#gmail.com"
},
"321123": {
"id": 321123,
"first_name": "John",
"last_name": "Wayne",
"username": "notreal#email.com",
"email": "notreal#email.com"
},
"98765": {
"id": 98765,
"first_name": "Suzie",
"last_name": "Q",
"username": "email#company.com",
"email": "email#company.com"
}
}
},
"more": false
}
Instead of an array of users, each user is listed as a separate property with the id as the name of the property. They use this pattern on all the endpoints, so I need a way to know what the structure of the response is going to be in order to query it like I'm used to.
I've written a statement that uses dynamic sql to get this into a table structure, but I was wondering if someone more skilled with the JSON functions could propose a better solution.
Here's my SQL code...
GO
--// simplifed version of the actual json response for security and demo purposes
DECLARE #user_response NVARCHAR(MAX) = N'
{
"results": {
"users": {
"12345": {
"id": 12345,
"first_name": "Demo",
"last_name": "User",
"username": "demo#gmail.com",
"email": "demo#gmail.com"
},
"321123": {
"id": 321123,
"first_name": "John",
"last_name": "Wayne",
"username": "notreal#email.com",
"email": "notreal#email.com"
},
"98765": {
"id": 98765,
"first_name": "Suzie",
"last_name": "Q",
"username": "email#company.com",
"email": "email#company.com"
}
}
},
"more": false
}
'
--// put users object into variable
DECLARE #users NVARCHAR(MAX) = (
SELECT users.users
FROM OPENJSON(#user_response)
WITH (results NVARCHAR(MAX) AS JSON
, more VARCHAR(20)) as body
CROSS APPLY OPENJSON(results)
WITH (users NVARCHAR(MAX) AS JSON) as users
)
--// extract the keys from the users object
DECLARE #keys TABLE ([key] VARCHAR(100), [index] INT)
INSERT INTO #keys
SELECT [key], ROW_NUMBER() OVER (ORDER BY [key]) 'index'
FROM OPENJSON(#users)
--// initialize looping variables
DECLARE #i INT = 1
, #key VARCHAR(100)
, #sql NVARCHAR(MAX)
SELECT #sql = 'DECLARE #user_response NVARCHAR(MAX) = N''' + #user_response + ''''
--// loop through keys and UNION individual queries on the original json response
WHILE (#i <= (SELECT MAX([index]) FROM #keys))
BEGIN
SELECT #key = (SELECT [key] FROM #keys WHERE [index] = #i)
SELECT #sql = #sql + CASE WHEN #i = 1 THEN '' ELSE 'UNION' END + '
SELECT user_data.*
FROM OPENJSON(#user_response)
WITH (results NVARCHAR(MAX) AS JSON)
CROSS APPLY OPENJSON(results)
WITH (users NVARCHAR(MAX) AS JSON)
CROSS APPLY OPENJSON(users)
WITH ([' + #key + '] NVARCHAR(MAX) AS JSON)
CROSS APPLY OPENJSON([' + #key + '])
WITH (id INT
, first_name VARCHAR(100)
, last_name VARCHAR(100)
, username VARCHAR(200)
, email VARCHAR(200)) as [user_data]'
SELECT #i = #i + 1
END
--// execute final dynamic query
EXEC sp_executesql #sql
The resultset of this statement looks like this:
|id |first_name|last_name|username |email |
|-----|----------|---------|-----------------|-----------------|
|98765|Suzie |Q |email#company.com|email#company.com|
|321123|John |Wayne |notreal#email.com|notreal#email.com|
|12345|Demo |User |demo#gmail.com |demo#gmail.com |
Thanks in advance for your ideas and feedback.

Like this:
select u.*
from openjson(#user_response, '$.results.users') d
cross apply
openjson(d.value)
with
(
id int '$.id',
first_name varchar(200) '$.first_name',
last_name varchar(200) '$.last_name',
username varchar(200) '$.username',
email varchar(200) '$.email'
) u
if you need it you can get the object name with d.[key]

Related

SQL Server : I want to access Json Array of object

DECLARE #json varchar(max),
#errors varchar(max),
#policy_number varchar(10)
SET #json =
'{
"returnMessage": "",
"policy_number": "12345",
"documents": {
"policy_document": "",
"tax_invoice_document": ""
},
"errors": [
{
"error_code": "999",
"error_message1": "Error"
}
]
}'
I want to get Error_code, error_message1
SELECT
#policy_number = policy_number,
#errors = errors
FROM
OPENJSON(#json) WITH
(
policy_number VARCHAR(10) '$.policy_number',
errors VARCHAR(max) '$.errors'
)
If you only want data from the errors property, you can go straight to that with a single OPENJSON call
DECLARE #json varchar(max)
SET #json =
'{
"returnMessage": "",
"policy_number": "12345",
"documents": {
"policy_document": "",
"tax_invoice_document": ""
},
"errors": [
{
"error_code": "999",
"error_message1": "Error"
}
]
}'
SELECT
policy_number = JSON_VALUE(#json, '$.policy_number'),
error_code,
error_message1
FROM
OPENJSON(#json, '$.errors')
WITH (
error_code VARCHAR(100),
error_message1 VARCHAR(100)
);
You're close. You need AS JSON in your OPENJSON and then you can use CROSS APPLY to pull your desired values out of errors.
declare #json varchar(max),#errors varchar(max),#policy_number varchar(10)
set #json =
'{
"returnMessage": "",
"policy_number": "12345",
"documents": {
"policy_document": "",
"tax_invoice_document": ""
},
"errors": [
{
"error_code": "999",
"error_message1": "Error"
}
]
}';
SELECT error_code, error_message1
FROM OPENJSON(#json)
WITH (errors NVARCHAR(MAX) AS JSON) l1
CROSS APPLY OPENJSON(l1.errors)
WITH (
error_code VARCHAR(100),
error_message1 VARCHAR(100)
);

SQL Server : build JSON dynamic key

I'm trying to create object with dynamic key and value with SQL Server. Currently I have the following query result:
[
{ "Name": "Ashley", "type": "human" },
{ "Name": "Dori", "type": "cat" }
]
and I am looking to build this:
[ { "Human": "Ashley" }, { "Cat": "Dori" } ]
PS: I'm not sure if it's even possible, but I have no ideas left.
If you know every possible combination up front, you can specify them in CASE expressions like this
SELECT
Human = CASE WHEN t.[type] = 'human' THEN t.Name END,
Cat = CASE WHEN t.[type] = 'cat' THEN t.Name END
-- more options
FROM YourTable t
FOR JSON PATH;
If you don't know what the possible options are, you will need dynamic SQL.
Try to keep a clear head about which part is static and which is dynamic. Use PRINT #sql; to test.
DECLARE #sql nvarchar(max) = '
SELECT
' + (
SELECT STRING_AGG(CAST(
' ' + typeCol + ' = CASE WHEN t.[Type] = ' + typeStr + ' THEN t.Name END'
AS nvarchar(max)), ',
')
FROM (
SELECT DISTINCT
typeCol = QUOTENAME(UPPER(LEFT(t.[type], 1)) + SUBSTRING(t.[type], 2, LEN(t.[type]))),
typeStr = QUOTENAME(t.[type], '''')
FROM YourTable t
)
) + '
FROM YourTable t
FOR JSON PATH;
';
PRINT #sql; --for testing
EXEC sp_executesql #sql; -- add parameters if necessary
I would like to add another solution, perhaps for different cases.
In case you already have a generated JSON and you want to transform it, for example:
[
{
"key": "key1",
"value": "value1"
},
{
"key": "key2",
"value": "value2"
},
{
"key": "key3",
"value": "value3"
}
]
You can apply this sql script to transform it:
SET #json = REPLACE(#json, '","', '":"')
SET #json = REPLACE(#json, '"key":', '')
SET #json = REPLACE(#json, '"value":', '')
SET #json = REPLACE(#json, '{', '')
SET #json = REPLACE(#json, '}', '')
SET #json = REPLACE(#json, '[', '{')
SET #json = REPLACE(#json, ']', '}')
for the example, the result would transform into this:
{
"key1": "value1",
"key2": "value2",
"key3": "value3"
}

TSQL - Pivot not returning all rows

I have the following JSON array data set that needs to be parsed into 2 table rows:
[
{ "eid": "ABCDGD",
"name": "Carol E",
"email": "carole#gmail.com",
"role": "Recruiter"
},
{ "eid": "HDHDK",
"name": "Mark H",
"email": "markh#gmail.com",
"role": "Manager"
}
]
I need the code below to return both sets of employee information but it only returns one. How do I achieve this?
select p.* from
(SELECT j2.[key] as _keys, j2.Value as _vals
FROM OPENJSON(#c) j1
CROSS APPLY OPENJSON(j1.Value) j2
) as pds
PIVOT
(
max(pds._vals)
FOR pds._keys IN([eid], [name], [email], [role])
) AS p
SQLfiddle - http://sqlfiddle.com/#!18/9eecb/54970
No need to pivot, just specify your json columns and will give your desired results.
SELECT *
FROM OPENJSON(#c) WITH (
eid varchar(200) '$.eid',
name varchar(200) '$.name',
email varchar(200) '$.email',
role varchar(200) '$.role'
) j1
JSON is already maintaining a table kind structure in it and can be directly converted into table by using OPENJSON.
As the syntax of OPENJSON on MSDN website.
OPENJSON( jsonExpression [ , path ] ) [ <with_clause> ]
<with_clause> ::= WITH ( { colName type [ column_path ] [ AS JSON ] } [ ,...n ] )
Here just need to pass your column name as maintained in JSON and will convert your JSON into SQL Table.
You may find more details on this link.
For your above query you may try this.
DECLARE #json NVARCHAR(MAX)
SET #json =
N'[
{ "eid": "ABCDGD",
"name": "Carol E",
"email": "carole#gmail.com",
"role": "Recruiter"
},
{ "eid": "HDHDK",
"name": "Mark H",
"email": "markh#gmail.com",
"role": "Manager"
}
]'
SELECT *
FROM OPENJSON(#json)
WITH (
eid nvarchar(50) '$.eid',
name nvarchar(50) '$.name',
email nvarchar(50) '$.email',
role nvarchar(50) '$.role',
)

Nest Json Array merge

I have a column saved json data in my table:
declare #json nvarchar(max)
set #json = N'
{
"Companies": [
{
"CompanyId": "A",
"Employee": null
},
{
"CompanyId": "B",
"Employee": [
{
"EmployeePictureId": null,
"Name": "Employee1"
},
{
"EmployeePictureId": "PictureId2",
"Name": "Employee2"
}
]
},
{
"CompanyId": "C",
"Employee": [
{
"EmployeePictureId": null,
"Name": "Employee3"
},
{
"EmployeePictureId": null,
"Name": "Employee4"
}
]
}
]
}
'
Is it posible to get the result like:
{
"EmployeePictureIds": ["PictureId2"]
}
using the Json_Query, Json_Value, OPENJSON...
Only get EmployeePictureId and skip empty(null) data
By the way, the count of elements in array are not sure.
In SQL Server 2017 you can use the following query:
select json_query(QUOTENAME(STRING_AGG('"' + STRING_ESCAPE( A.EmployeePictureId , 'json')
+ '"', char(44)))) as [EmployeePictureIds]
FROM OPENJSON(#json, '$.Companies')
WITH
(
CompanyId NVARCHAR(MAX),
Employee NVARCHAR(MAX) as json
) as B
cross apply openjson (B.Employee)
with
(
EmployeePictureId VARCHAR(50),
[Name] VARCHAR(50)
) as A
where A.EmployeePictureId is not null
for json path , WITHOUT_ARRAY_WRAPPER
Results for the JSON you provided:
Results adding another non null EmployeePictureId:

How to reference array items of json object in T-SQL

I am using the JSON functionality in SQL Server 2016. How do I reference an item in the $.people[] array as shown below, using a variable? Instead of hardcoding the "1" in the path parameter of the JSON_QUERY function, I want to use a variable and loop through each item in the people array.
declare #json nvarchar(max) = '{
"people": [{
"name": "John",
"surname": "Doe"
}, {
"name": "Jane",
"surname": null,
"active": true
}]
}';
select JSON_QUERY(#json,'$.people[1]'); -- this works
declare #test nvarchar(max) = '$.people[1]';
select JSON_QUERY(#json,#test); -- ERROR: The argument 2 of the "JSON_VALUE or JSON_QUERY" must be a string literal.
You can use dynamic SQL through the sp_executesql procedure.
Both the original select statement, and the sp_executesql have the same output listed at the bottom.
declare #json nvarchar(max) = '{
"people": [{
"name": "John",
"surname": "Doe"
}, {
"name": "Jane",
"surname": null,
"active": true
}]
}';
declare #i int = 1
declare #dyn_sql nvarchar(1000) =
'select json_query(#json_dyn, ''$.people[' + cast(#i as varchar(10)) + ']'');'
select JSON_QUERY(#json,'$.people[1]'); -- this works
exec sp_executesql #dyn_sql --set the SQL to run
, N'#json_dyn nvarchar(max)' --"declare" the variable in the SQL to run
, #json_dyn = #json --"set" the variable in the SQL to run
/*
Output : { "name": "Jane", "surname": null, "active": true }
*/

Resources