How to target and count value with JQ? - arrays

From this file :
[
{
"network": "X.X.X.1",
"defaultGateway": "X.X.X.X",
"ipAddressTab": [
{
"foo1": "10.0.0.1",
"foo2": "network",
"status": "reserved",
"foo4": null,
"foo5": null,
"foo6": null,
"foo7": null,
"foo8": null,
"foo9": null,
"foo10": null,
"foo11": null
},
{
"foo1": "10.0.0.2",
"foo2": "network",
"status": "reserved",
"foo4": null,
"foo5": null,
"foo6": null,
"foo7": null,
"foo8": null,
"foo9": null,
"foo10": null,
"foo11": null
},
{
"foo1": "10.0.0.3",
"foo2": "network",
"status": "reserved",
"foo4": null,
"foo5": null,
"foo6": null,
"foo7": null,
"foo8": null,
"foo9": null,
"foo10": null,
"foo11": null
},
{
"foo1": "10.0.0.4",
"status": "available"
},
{
"foo1": "10.0.0.5",
"status": "available"
},
{
"foo1": "10.0.0.6",
"status": "available"
},
{
"foo1": "10.0.0.7",
"status": "available"
}
],
"full": false,
"id": "xxx"
},
{
"network": "X.X.X.2",
"defaultGateway": "X.X.X.X",
"ipAddressTab": [
{
"foo1": "10.0.0.1",
"foo2": "network",
"status": "reserved",
"foo4": null,
"foo5": null,
"foo6": null,
"foo7": null,
"foo8": null,
"foo9": null,
"foo10": null,
"foo11": null
},
{
"foo1": "10.0.0.2",
"foo2": "network",
"status": "reserved",
"foo4": null,
"foo5": null,
"foo6": null,
"foo7": null,
"foo8": null,
"foo9": null,
"foo10": null,
"foo11": null
},
{
"foo1": "10.0.0.3",
"foo2": "network",
"status": "reserved",
"foo4": null,
"foo5": null,
"foo6": null,
"foo7": null,
"foo8": null,
"foo9": null,
"foo10": null,
"foo11": null
},
{
"foo1": "10.0.0.4",
"status": "available"
},
{
"foo1": "10.0.0.5",
"status": "available"
},
{
"foo1": "10.0.0.6",
"status": "available"
},
{
"foo1": "10.0.0.7",
"status": "available"
}
],
"full": false,
"id": "xxx"
}
]
# Just an example, there is more lines in my file
I can keep the informations that I want :
cat myfile | jq 'map({network, full})'
[
{
"network": "X.X.X.1",
"full": false
},
{
"network": "X.X.X.2",
"full": false
}
]
Now I'm looking for a tip to count and display some values. For example, I would like to display the number of reserved, allocated and available like that :
[
{
"network": "X.X.X.1",
"full": false
reserved: 3
available: 4
},
{
"network": "X.X.X.2",
"full": false
reserved: 3
available: 4
}
]
I've look everywhere and I found no good example to do that...
Some one to show me how can I have this output ?
Thanks !

Use reduce to count statuses.
map({network, full} +
reduce .ipAddressTab[].status as $s ({}; .[$s] += 1))
Online demo
You can change {} to {reserved: 0, available: 0} to maintain a consistent order of keys among all the entries.

One way to do this would be to use a function to count the objects of each type
def f($path; $val): $path | map(select(.status == $val)) | length;
map({network, full, reserved: f(.ipAddressTab; "reserved"), available: f(.ipAddressTab; "available")})
jqplay - Demo
The function f takes a path and the status string to be looked-up then gets the length of the objects in the array.
With oguz ismail's suggestion to avoid repetition
def f($val): map(select(.status == $val)) | length;
map({network, full} + (.ipAddressTab | { reserved: f("reserved"), available: f("available")}))

When counting a potentially large number of objects, it's usually better to
use stream-oriented filters so as to avoid constructing arrays. These two are often useful as a pair, though in the present case defining just count/1 by itself would be sufficient:
def sigma(s): reduce s as $x (0; .+$x);
def count(s): sigma(s|1);
To achieve the stated goal, one can now simply write the specification as a program:
map({network,
full,
reserved: count(.ipAddressTab[] | select(.status == "reserved")),
available: count(.ipAddressTab[] | select(.status == "available"))
})
Generalization
Now for a little jq magic -- no references to specific "status" values at all:
def countStatus($s):
{($s): count(.ipAddressTab[] | select(.status == $s))};
def statuses: [.ipAddressTab[].status] | unique;
map( {network, full}
+ ([countStatus(statuses[])] | add) )
total_available
In a comment, a question about showing total_available was asked.
To add a total_available key to each object, you could append the
following to either of the above pipelines:
| {total_available: sigma(.[] | .available)} as $total
| map(. + $total)

Related

extract value from subarray and combine with upper value using jq to csv

I have a json file with a sub-array and I want to get each sub-array with the same id in separate lines.
json:
{
"success": true,
"status": {
"http": {
"code": 200,
"message": "OK"
}
},
"result": [{
"id": "123456789",
"start_date": "2021-01-01 08:17:39.989",
"snippets": [{
"transcript": "yes",
"matched_entry": null,
"start": "2021-01-16 11:32:25.922"
}, {
"transcript": null,
"matched_entry": null,
"start": "2021-01-16 11:32:38.179"
}]
}, {
"id": "987654321",
"start_date": "2021-01-01 08:17:39.989",
"duration_total": 301,
"snippets": [{
"transcript": "yes",
"matched_entry": null,
"start": "2021-01-01 08:17:54.055"
}, {
"transcript": "something",
"matched_entry": " meta entry",
"start": "2021-01-01 08:18:11.028"
}, {
"transcript": "no",
"matched_entry": null,
"start": "2021-01-01 08:18:24.057"
}]
}]
}
I try to get:
123456789, yes , null, "2021-01-16 11:32:25.922"
123456789, null, null, "2021-01-16 11:32:38.179"
987654321, yes, null, "2021-01-01 08:17:54.055"
987654321, something, "meta entry", "2021-01-01 08:18:11.028"
987654321, no, null, "2021-01-01 08:18:24.057"
first attempt was:
jq -rc ".result[] | {id: .id, snippetsTranscript: .snippets[].transcript, snippetsMatchedEntry: .snippets[].matched_entry, snippetsStart: .snippets[].start}" 210101_210121_copy.json
but the result is that every combination was returned:
{"id":"123456789","snippetsTranscript":"yes","snippetsMatchedEntry":null,"snippetsStart":"2021-01-16 11:32:25.922"}
{"id":"123456789","snippetsTranscript":"yes","snippetsMatchedEntry":null,"snippetsStart":"2021-01-16 11:32:38.179"}
{"id":"123456789","snippetsTranscript":"yes","snippetsMatchedEntry":null,"snippetsStart":"2021-01-16 11:32:25.922"}
{"id":"123456789","snippetsTranscript":"yes","snippetsMatchedEntry":null,"snippetsStart":"2021-01-16 11:32:38.179"}
{"id":"123456789","snippetsTranscript":null,"snippetsMatchedEntry":null,"snippetsStart":"2021-01-16 11:32:25.922"}
{"id":"123456789","snippetsTranscript":null,"snippetsMatchedEntry":null,"snippetsStart":"2021-01-16 11:32:38.179"}
{"id":"123456789","snippetsTranscript":null,"snippetsMatchedEntry":null,"snippetsStart":"2021-01-16 11:32:25.922"}
{"id":"123456789","snippetsTranscript":null,"snippetsMatchedEntry":null,"snippetsStart":"2021-01-16 11:32:38.179"} ...
the second attempt was:
jq -rc ".result[] | {id: .id, snippetsMatchedEntry: [.snippets[].matched_entry], snippetsStart: [.snippets[].start], snippetsTranscript: [.snippets[].transcript]}" 210901_210921_copy.json
but the result is:
{"id":"123456789","snippetsMatchedEntry":[null,null],"snippetsStart":["2021-01-16 11:32:25.922","2021-01-16 11:32:38.179"],"snippetsTranscript":["yes",null]}
{"id":"987654321","snippetsMatchedEntry":[null," meta entry",null],"snippetsStart":["2021-01-01 08:17:54.055","2021-01-01 08:18:11.028","2021-01-01 08:18:24.057"],"snippetsTranscript":["yes","something","no"]}
Is this possible with jq?
For each element of the result array, create an object with just the id field, and for each element of the snippets sub-array, add it:
.result[] | {id} + .snippets[]
If you don't need all fields of the snippets array, simply declare them as before
.result[] | {id} + (.snippets[] | {transcipt, matched_entry, start})
Try it on jqplay.org

Where can I find kiwi tcms parameters information about json-rpc?

I practice to use json-rpc to create test case, and I want to assoicate a test paln with test case, but I don't know the parameter of the plan.
Can anyone give me some suggestions?? Thanks.
My example like this
Test plan ID : 3
Test plan name: test
Using postman request
{
"jsonrpc":"2.0",
"method":"TestCase.create",
"params":{"values":{"summary":"jsonrpctest","case_status":2,"category":2,"priority":1,"text":"20201005test","plan":[3,"test"]}},
"id":1
}
Response
{
"id": 1,
"jsonrpc": "2.0",
"result": {
"id": 191,
"create_date": "2020-10-06 04:44:13",
"is_automated": false,
"script": "",
"arguments": "",
"extra_link": null,
"summary": "jsonrpctest",
"requirement": null,
"notes": "",
"text": "20201005test",
"case_status_id": 2,
"case_status": "CONFIRMED",
"category_id": 2,
"category": "--default--",
"priority_id": 1,
"priority": "P1",
"author_id": 1,
"author": "ardyn",
"default_tester_id": null,
"default_tester": null,
"reviewer_id": null,
"reviewer": null,
"plan": [],
"component": [],
"tag": []
}
}
https://kiwitcms.readthedocs.io/en/latest/api/index.html says
"Server side RPC methods are documented in tcms.rpc.api."
Which is
https://kiwitcms.readthedocs.io/en/latest/modules/tcms.rpc.api.html
And there is the TestPlan.add_case() method:
https://kiwitcms.readthedocs.io/en/latest/modules/tcms.rpc.api.testplan.html#tcms.rpc.api.testplan.add_case

Validate an array of objects in Postman

I am quite new with Postman and not a particularly good programmer. I am testing an API and trying to validate a part of a response that has an array in it looking like this:
"processors": [
{
"name": "ARTPEC-5",
"type": "SOC",
"url": null,
"releaseNotes": null,
"cdnUrl": null,
"cdnReleaseNotes": null
},
{
"name": "SSL",
"type": "SOC",
"url": null,
"releaseNotes": null,
"cdnUrl": null,
"cdnReleaseNotes": null
},
{
"name": "ARTPEC-7",
"type": "SOC",
"url": null,
"releaseNotes": null,
"cdnUrl": null,
"cdnReleaseNotes": null
}
]
Now, I would like to validate that the array comes with the above objects. They may come in any order in the array, so I cannot refer to the objects using index like jsonData.processors[0] and then validate them one by one like that. I need a general validation method. I have tried this that did not work:
pm.test("Check if the response has processors", function () {
pm.expect(jsonData.processors).to.have.members([
{
"name": "ARTPEC-5",
"type": "SOC",
"url": null,
"releaseNotes": null,
"cdnUrl": null,
"cdnReleaseNotes": null
},
{
"name": "SSL",
"type": "SOC",
"url": null,
"releaseNotes": null,
"cdnUrl": null,
"cdnReleaseNotes": null
},
{
"name": "ARTPEC-7",
"type": "SOC",
"url": null,
"releaseNotes": null,
"cdnUrl": null,
"cdnReleaseNotes": null
}]);
});
This approach only gives me the cryptic error message AssertionError: expected [ Array(3) ] to have the same members as [ Array(3) ]
By using _.differenceWith() you will get empty array if there is no difference in the array objects and their properties. That you can assert as -
var _ = require('lodash')
var objects = [{ 'x': 1, 'y': 2 }, { 'x': 1, 'y': 2 }];
var arr = _.differenceWith(objects, [{ 'x': 1, 'y': 2 }], _.isEqual);
console.log(arr);
console.log(objects.length);
// assert that difference should be empty
pm.test("array difference", function () {
pm.expect([]).to.eql(arr)
});

Service Discovery versus DCOS Overlay Network

I've setup a DCOS 1.8 cluster and am currently familiarizing.
So far I have marathon-lb working like a charm with Jenkins via Host networking. Now I am trying to set things up using Overlay.
I have a couple of test containers, some in the dcos overlay network, some not. So far they can reach each other via IP, which is nice. However when I try to resolv containers on the overlay network using mesos-dns, all it resolves is the host address (not exactly what I am expecting).
So I played around some with marathon to figure it out. What I did was add a discovery block to ipAddress:
{
"volumes": null,
"id": "/mariadb10",
"cmd": null,
"args": null,
"user": null,
"env": {
"MYSQL_ROOT_PASSWORD": "foo"
},
"instances": 1,
"cpus": 1,
"mem": 1024,
"disk": 0,
"gpus": 0,
"executor": null,
"constraints": null,
"fetch": null,
"storeUrls": null,
"backoffSeconds": 1,
"backoffFactor": 1.15,
"maxLaunchDelaySeconds": 3600,
"container": {
"docker": {
"image": "mariadb:10.0",
"forcePullImage": false,
"privileged": false,
"network": "USER"
},
"type": "DOCKER",
"volumes": [
{
"containerPath": "/var/lib/mysql",
"hostPath": "/mnt/foo",
"mode": "RW"
}
]
},
"healthChecks": [
{
"protocol": "TCP",
"gracePeriodSeconds": 30,
"intervalSeconds": 10,
"timeoutSeconds": 10,
"maxConsecutiveFailures": 3,
"port": 3306
}
],
"readinessChecks": null,
"dependencies": null,
"upgradeStrategy": {
"minimumHealthCapacity": 1,
"maximumOverCapacity": 1
},
"labels": null,
"acceptedResourceRoles": null,
"ipAddress": {
"networkName": "dcos",
"discovery": {
"ports": [
{ "number": 3306, "name": "mysql", "protocol": "tcp" }
]
}
},
"residency": null,
"secrets": null,
"taskKillGracePeriodSeconds": null
}
Marathon tells me this is not allowed with "Bridge" or "User" networks. However it did not complain about the following and launched the container:
{
"volumes": null,
"id": "/mariadb10",
"cmd": null,
"args": null,
"user": null,
"env": {
"MYSQL_ROOT_PASSWORD": "foo"
},
"instances": 1,
"cpus": 1,
"mem": 1024,
"disk": 0,
"gpus": 0,
"executor": null,
"constraints": null,
"fetch": null,
"storeUrls": null,
"backoffSeconds": 1,
"backoffFactor": 1.15,
"maxLaunchDelaySeconds": 3600,
"container": {
"docker": {
"image": "mariadb:10.0",
"forcePullImage": false,
"privileged": false,
"network": "USER"
},
"type": "DOCKER",
"volumes": [
{
"containerPath": "/var/lib/mysql",
"hostPath": "/mnt/foo",
"mode": "RW"
}
]
},
"healthChecks": [
{
"protocol": "TCP",
"gracePeriodSeconds": 30,
"intervalSeconds": 10,
"timeoutSeconds": 10,
"maxConsecutiveFailures": 3,
"port": 3306
}
],
"readinessChecks": null,
"dependencies": null,
"upgradeStrategy": {
"minimumHealthCapacity": 1,
"maximumOverCapacity": 1
},
"labels": null,
"acceptedResourceRoles": null,
"ipAddress": {
"networkName": "dcos"
},
"residency": null,
"secrets": null,
"taskKillGracePeriodSeconds": null
}
Funny thing is, it does not use the overlay address anymore, but now listens to the hosts address and also announces the hosts address into the overlay network.
Am I just doing it wrong or does that not work as expected, yet?
So,
I found the solution myself. The easy workaround is to edit /opt/mesosphere/etc/mesos-dns.json. Then change the order of IPSources to list netinfo first.
For more information, you can also check here
Alternatively, you can use taskname.marathon.containerip.dcos.thisdcos.directory. It is documented here: https://docs.mesosphere.com/1.8/administration/overlay-networks/.

AngularJS Recursive Data Structure Traversal with Unique Keys and Arrays

I believe this question is unique in that I am attempting to traverse a data structure containing objects with unique keys and arrays. I've seen other questions regarding this subject, but they reference simple data structures with multiple levels the same object.
Ultimately, I would like to be able to generate a form from any given data structure simply by traversing it as an object or an array.
I've created two directives, one for traversing an object and the other for traversing an array. Basically I'm trying to pass in the array or object as a parameter into the directive's isolated scope, then generate the associated form fields by referencing the scope variable scope.context. I'm not getting any errors, but I'm also not getting the expected result.
Am I approaching this the right way?
Any thoughts or suggestions would be greatly appreciated.
HTML
<div ng-app="app" ng-controller="CreateFormController">
<div traverse-object="{{model.form}}"></div>
<script type="text/ng-template" id="traverseObject.tmpl.html">
<div ng-repeat="(key, value) in context">
<div ng-switch="getTypeOf(value)">
<div ng-switch-when="string">
{{key}} string
</div>
<div ng-switch-when="null">
{{key}} null
</div>
<div ng-switch-when="object">
{{key}} object
<div traverseObject = "{{value}}"></div>
</div>
<div ng-switch-when="array">
{{key}} array
<div traverseArray = "{{value}}"></div>
</div>
<div ng-switch-default="true">
{{key}} other
</div>
</div>
</div>
</script>
<script type="text/ng-template" id="traverseArray.tmpl.html">
<div ng-repeat="item in context">
<div ng-switch="getTypeOf(item)">
<div ng-switch-when="string">
{{item}}
</div>
<div ng-switch-when="null">
{{item}}
</div>
<div ng-switch-when="object">
<div traverseObject = "{{item}}"></div
</div>
<div ng-switch-when="array">
<div traverseArray = "{{item}}"></div>
</div>
<div ng-switch-default="true">
{{item}}
</div>
</div>
</div>
</script>
</div>
JS
var json = {
"ProjectName": null,
"ProjectNumber": null,
"BillingAU": null,
"BillingMailCode": null,
"BorrowersName": null,
"Comment": null,
"CompanyNumber": null,
"DesiredDeliveryDate": null,
"LendingGroup": "",
"LendingOffice": null,
"LoanAmount": null,
"LoanAmountOther": null,
"LoanCloseDate": null,
"LoanNumber": null,
"LoanTerm": null,
"Obligor": null,
"OriginatingAU": null,
"PreviousProjectNumber": null,
"PreviousReport": null,
"TransactionComments": null,
"IsInvolvement": null,
"InvolvementType": "",
"IsFFESecured": null,
"IsJuniorLien": null,
"IsParticipationLoan": null,
"IsWithCrossCollateral": null,
"IsWithLoanGuarantor": null,
"TotalParticipationLoanAmount": null,
"LienPosition": null,
"LoanExist": "",
"LoanType": "",
"OtherLien": null,
"PriorityApproval": "",
"ProjectPurpose": "",
"LoanTransType": "",
"ServiceRequestPeople": [
{
"PersonType": "Account Officer",
"FirstName": null,
"LastName": null,
"Email": null,
"Company": null,
"CompanyNumber": null,
"ImportUserId": null,
"PhoneNumber": "",
"OtherNumber": "",
"FaxNumber": "",
"Address": {
"StreetAddress": "",
"StreetAddress2": "",
"City": "",
"State": "",
"ZipCode": "",
"County": "",
"Nation": "United States",
"Links": []
},
"Links": []
}
],
"Properties": [
{
"DevelopmentName": null,
"PropertyTypesMajor": "",
"PropertyTypes": "",
"PropertyAddress": {
"StreetAddress": "",
"StreetAddress2": "",
"City": "",
"State": "",
"ZipCode": "",
"County": "",
"Nation": "United States",
"Links": []
},
"PropertyDescription": null,
"PropertyCondition": "",
"PropertyRemark": null,
"PropertyStatus": "",
"PropertyTenancy": "",
"ExternalReferenceNumber": null,
"FhaCaseNumber": null,
"LegalComment": null,
"NumberOfBuildings": null,
"NumberOfTenants": null,
"NumberOfUnits": null,
"Occupancy": null,
"PriorSaleDate": "",
"PriorSalePurchaseAmount": null,
"TotalPurchaseAmount": null,
"ProposedUse": null,
"CurrentUse": null,
"Stories": null,
"YearBuilt": null,
"YearLastRenovate": null,
"RentableArea": null,
"Zoning": null,
"ZoningCodeDescription": null,
"MapNumber": null,
"IsForSale": null,
"ListSaleAmount": null,
"IsLegalDescription": null,
"IsPendingSale": null,
"PendingSaleAmount": null,
"PendingSaleDate": "",
"IsRenovationDemo": null,
"RenovationDemoDescription": null,
"IsFloodPlain": null,
"FloodPlainDescription": null,
"FloodPlainSurveyed": null,
"IsGroundLease": null,
"ParkingType": [
""
],
"PrimaryImprovement": {
"Measurement": 0,
"Unit": "",
"Links": []
},
"SecondaryImprovement": {
"Measurement": 0,
"Unit": "",
"Links": []
},
"LandSize": {
"Measurement": 0,
"Unit": "",
"Links": []
},
"ExcessLandSize": {
"Measurement": 0,
"Unit": "",
"Links": []
},
"PropertyParcel": [
{
"ParcelNumber": null,
"Size": 0,
"Links": []
}
],
"PropertyPeople": [
{
"PersonType": "",
"FirstName": null,
"LastName": null,
"Email": null,
"PhoneNumber": "",
"OtherNumber": "",
"FaxNumber": "",
"Affiliation": null,
"Links": []
}
],
"Services": [
{
"ServiceType": null,
"Comments": null,
"SentDate": "",
"DesiredDeliveryDate": "",
"Links": []
}
],
"Links": []
}
],
"Links": []
}
var app = angular.module('app', [])
app.controller('CreateFormController', function($scope){
$scope.model = {}
$scope.model.form = json
})
app.directive("traverseObject", function(){
return {
scope: {
traverseObject: '#'
},
templateUrl: 'traverseObject.tmpl.html',
link: function(scope){
scope.getTypeOf = function(value){
if(value === null) {
return "null";
}
if(Array.isArray(value)) {
return "array";
}
return typeof value;
}
scope.context = JSON.parse(scope.traverseObject)
}
}
})
app.directive("traverseArray", function(){
return {
scope: {
traverseArray: '#'
},
templateUrl: 'traverseArray.tmpl.html',
link: function(scope){
scope.getTypeOf = function(value){
if(value === null) {
return "null";
}
if(Array.isArray(value)) {
return "array";
}
return typeof value;
}
scope.context = JSON.parse(scope.traverseArray)
}
}
})

Resources