How can I "shred" JSON data already in a SQL database? - sql-server

Question

I have successfully imported a JSON file into a SQL 2016 database and I am now attempting to parse that data so that I can populate a table for each of the fields in the column that is holding the JSON data. I am no DBA and it took me a couple of days to figure out how to successfully import this data. I need to know how to accomplish this using SQL. I am not certain what other information I need to provide here, so if anything else is needed, let me know and I will provide that info.
The table name is dbo.IncapsulaSourceData. This table has 5 columns: Site_ID, JSON_Source, Processed, Date_inserted, Date_processed.
Here is a sample of the JSON data that is being stored in the JSON_Source column:
{
"site_id":123456,
"statusEnum":"fully_configured",
"status":"fully-configured",
"domain":"site.name.com",
"account_id":111111,
"acceleration_level":"standard",
"site_creation_date":1410815844000,
"ips":[
"99.99.99.99"
],
"dns":[
{
"dns_record_name":"site.name.com",
"set_type_to":"CNAME",
"set_data_to":[
"frgt.x.wafdns.net"
]
}
],
"original_dns":[
{
"dns_record_name":"name.com",
"set_type_to":"A",
"set_data_to":[
""
]
},
{
"dns_record_name":"site.name.com",
"set_type_to":"A",
"set_data_to":[
"99.99.99.99"
]
},
{
"dns_record_name":"site.name.com",
"set_type_to":"CNAME",
"set_data_to":[
""
]
}
],
"warnings":[
],
"active":"active",
"additionalErrors":[
],
"display_name":"site.name.com",
"security":{
"waf":{
"rules":[
{
"action":"api.threats.action.block_ip",
"action_text":"Block IP",
"id":"api.threats.sql_injection",
"name":"SQL Injection"
},
{
"action":"api.threats.action.block_request",
"action_text":"Block Request",
"id":"api.threats.cross_site_scripting",
"name":"Cross Site Scripting"
},
{
"action":"api.threats.action.block_ip",
"action_text":"Block IP",
"id":"api.threats.illegal_resource_access",
"name":"Illegal Resource Access"
},
{
"block_bad_bots":true,
"challenge_suspected_bots":true,
"exceptions":[
{
"values":[
{
"ips":[
"99.99.99.99"
],
"id":"api.rule_exception_type.client_ip",
"name":"IP"
}
],
"id":123456789
},
{
"values":[
{
"ips":[
"99.99.99.99"
],
"id":"api.rule_exception_type.client_ip",
"name":"IP"
}
],
"id":987654321
}
],
"id":"api.threats.bot_access_control",
"name":"Bot Access Control"
},
{
"activation_mode":"api.threats.ddos.activation_mode.auto",
"activation_mode_text":"Auto",
"ddos_traffic_threshold":1000,
"id":"api.threats.ddos",
"name":"DDoS"
},
{
"action":"api.threats.action.quarantine_url",
"action_text":"Auto-Quarantine",
"id":"api.threats.backdoor",
"name":"Backdoor Protect"
},
{
"action":"api.threats.action.block_ip",
"action_text":"Block IP",
"id":"api.threats.remote_file_inclusion",
"name":"Remote File Inclusion"
},
{
"action":"api.threats.action.disabled",
"action_text":"Ignore",
"id":"api.threats.customRule",
"name":"wafRules"
}
]
},
"acls":{
"rules":[
{
"ips":[
"99.99.99.99"
],
"id":"api.acl.whitelisted_ips",
"name":"Visitors from whitelisted IPs"
},
{
"geo":{
"countries":[
"BR",
"NL",
"PL",
"RO",
"RU",
"TR",
"TW",
"UA"
]
},
"id":"api.acl.blacklisted_countries",
"name":"Visitors from blacklisted Countries"
}
]
}
},
"sealLocation":{
"id":"api.seal_location.none",
"name":"No seal "
},
"ssl":{
"origin_server":{
"detected":true,
"detectionStatus":"ok"
},
"generated_certificate":{
"ca":"GS",
"validation_method":"email",
"validation_data":"administrator#site.name.com",
"san":[
"*.site.name.com"
],
"validation_status":"done"
}
},
"siteDualFactorSettings":{
"specificUsers":[
],
"enabled":false,
"customAreas":[
],
"allowAllUsers":true,
"shouldSuggestApplicatons":true,
"allowedMedia":[
"ga",
"sms"
],
"shouldSendLoginNotifications":true,
"version":0
},
"login_protect":{
"enabled":false,
"specific_users_list":[
],
"send_lp_notifications":true,
"allow_all_users":true,
"authentication_methods":[
"ga",
"sms"
],
"urls":[
],
"url_patterns":[
]
},
"performance_configuration":{
"advanced_caching_rules":{
"never_cache_resources":[
],
"always_cache_resources":[
]
},
"acceleration_level":"standard",
"async_validation":true,
"minify_javascript":true,
"minify_css":true,
"minify_static_html":true,
"compress_jepg":true,
"progressive_image_rendering":false,
"aggressive_compression":false,
"compress_png":true,
"on_the_fly_compression":true,
"tcp_pre_pooling":true,
"comply_no_cache":false,
"comply_vary":false,
"use_shortest_caching":false,
"perfer_last_modified":false,
"accelerate_https":false,
"disable_client_side_caching":false,
"cache300x":false,
"cache_headers":[
]
},
"extended_ddos":1000,
"res":0,
"res_message":"OK",
"debug_info":{
"id-info":"1234"
}
}

Answer 1

Here is an idea of how to parse top level and second level data from json:
select top 100
ids.Site_ID,
ids.JSON_Source,
ids.Processed,
ids.Date_inserted,
ids.Date_processed,
x.site_id,
x.statusEnum,
x.[status],
x.[domain],
x.account_id,
x.acceleration_level,
x.site_creation_date,
x.ips as ipsJson,
x.dns as dnsJson,
ipsArr.key as ipKey,
ipsArr.value as [ip],
dnsArr.dns_record_name,
dnsArr.set_type_to,
dnsArr.set_data_to
from IncapsulaSourceData isd
outer apply openjson(isd.JSON_Source)
with (
site_id bigint,
statusEnum nvarchar(max),
[status] nvarchar(max),
[domain] nvarchar(max),
account_id bigint,
acceleration_level nvarchar(max),
site_creation_date bigint,
ips nvarchar(max) as json,
dns nvarchar(max) as json
-- JSON_Source contains only 1 object, so original rows
-- won't be duplicated. But this object contains some arrays
-- e.g. ips and dns. We don't parse them in this apply so no problem here
) as x
outer apply openjson(isd.JSON_Source, '$.ips') as ipsArr
-- here we parse arrays (ips above and dns below).
-- For some ids.row if there're 10 ipsArr values then in output
-- there'll be 10 rows <ids.row, ipsArr.value>.
-- I mean maybe you want to parse them in another query at all.
outer apply openjson(isd.JSON_Source, '$.dns')
with (
dns_record_name nvarchar(max),
set_type_to nvarchar(max),
set_data_to nvarchar(max) as json
) as dnsArr
-- other outer applies here
Sorry, wrote without checking in IDE, maybe made some mistakes.
p.s. maaaaaan, your inconsistent naming makes me cry!