I have a nested field with arrays in array in JSON like the following:
{
"foo": {
"bar": [
[
"a",
"b"
],
[
"c",
"d"
]
]
}
}
The following is my config file:
input {
file {
codec => "json"
path => "pathtofile"
type => "footype"
start_position => "beginning"
}
}
filter {
json {
source => "message"
remove_field => [ "host", "message", "path" ]
}
}
output {
elasticsearch {
action => "index"
index => "bar"
hosts => [ "http://localhost:9200" ]
}
}
I got the following error:
09:40:47.725 [[main]>worker0] WARN logstash.outputs.elasticsearch -
Failed action. {:status=>400, :action=>["index", {:_id=>nil,
:_index=>"bar", :_type=>"footype", :_routing=>nil},
2017-02-13T01:40:30.387Z myconnection %{message}],
:response=>{"index"=>{"_index"=>"bar", "_type"=>"footype",
"_id"=>"AVo1IN0vK2jgwdCXqZ-q", "status"=>400,
"error"=>{"type"=>"illegal_argument_exception", "reason"=>"mapper
[foo.bar] of different type, current_type [long], merged_type
[text]"}}}}
I have a feeling that it's the array problem. I have done some research and know that array is not well supported. But I need to ingest the array in elasticsearch. Is there a way to actually do that?
Any helps will be appreciated.
I solved this by using a ruby filter:
ruby {
code => '
j = 0
for i in event.get("[foo][bar]") do
#i is an array element in the big array
l = 0
for k in i do
event.set("item#" + j.to_s + "#" + l.to_s, k)
l = l + 1
end
j = j + 1
end
'
}
This will eventually produce fields
item#0#0 = "a"
item#0#1 = "b"
item#1#0 = "c"
item#1#1 = "d"
Related
let's suppose we have this variable:
foobars = {
"first" : {
specialkeys: [
"a",
"b",
"c"
]
}
"second" : {}
}
now let's say we would like to loop over that foobars object knowing that specialkeys doesn't exist in the "second" object.
This is what I tried but it complains that
This object does not have an attribute named specialkeys
My tries:
data = flatten([
for k, v in var.foobars : [
for sk in v.specialkeys : {
special = sk,
foo = k
}
]
])
I believe you would want to do the following:
data = flatten([
for k, v in local.foobars :
[
for sk, sv in v :
[
for spec in sv : {
special = spec,
foo = k
}
]
]
])
This will output something like this:
[
{
"foo" = "first"
"special" = "a"
},
{
"foo" = "first"
"special" = "b"
},
{
"foo" = "first"
"special" = "c"
},
]
I have a question:
(sorry for the bad formatting)
I have an array:
[
{
"data": [
[
"kek",
"lol"
],
[
"notkek",
"notlol"
]
]
}
]
If someone writes "kek" it should search it in the "data" array and return the "kek" position inside its array
(["kek","lol"])
and its array position
{
"data": [
[
"kek",
"lol"
]}
(in this case "data[0]")
If anybody knows the answer, please help me
The method Array.findIndex and Array.includes may help you
const obj = {
data: [
[
'kek',
'lol'
],
[
'notkek',
'notlol'
],
],
};
const keyToSearch = 'kek';
// We look for the key
const index = obj.data.findIndex(x => x.includes(keyToSearch));
if (index === -1) {
console.log(`We didn't found ${keyToSearch}`);
} else {
console.log(`We found ${keyToSearch} at index ${index}`);
}
Double index recuperation
const obj = {
data: [
[
'kek',
'lol'
],
[
'notkek',
'notlol'
],
[
'notkek',
'notlol',
'otherstuff',
'kek',
'test',
],
],
};
const keyToSearch = 'kek';
const ret = obj.data.reduce((tmp, x, xi) => {
// We look for the key
const index = x.findIndex(y => y === keyToSearch);
if (index === -1) return tmp;
return [
...tmp,
{
absoluteIndex: xi,
relativeIndex: index,
},
];
}, []);
if (!ret.length) {
console.log(`We didn't found ${keyToSearch}`);
} else {
ret.forEach(({
absoluteIndex,
relativeIndex,
}) => console.log(
`We found ${keyToSearch} at`,
`data index ${absoluteIndex}, in ${relativeIndex} position`,
));
}
userInput = 'kek'
let item = data.map((item, indx) => {
item.includes(userInput) ? return({"indx":indx,"nestedIndex":item.indexOf(userInput)}) : null
})
map over the data array and if the nested array had the item your searching for than return the index of the array and the index of the item with in that array
I have an array of hashes:
[
{
"June" => { "A" => { 3 => 48.4 } }
},
{
"January" => { "C" => { 2 => 88.0} }
},
{
"January"=> { "B" => { 2 => 44.0} }
},
{
"January"=> { "C" => { 4 => 48.8} }
}
]
I need to group each similar hash key into an array of the subsequent values like the following:
{
"June" => [{ "A" => [{ 3 => 48.4 }]] },
"January" => [
{ "B" => [{ 2 => 44.0}],
{ "C" => [{ 2 => 88.0}, { 4 => 48.8}],
] }
}
I am looking for an efficient method of grouping these elements. Can anyone help me master this hash of hashes?
I am trying to avoid looping through the base array and grouping manually. I was hoping that map (or some other enumerable method) might give what I want. When I used reduce(Hash.new, :merge), it came close but it used the last hash for each month key instead of adding it to an array.
Note: I added the following after gaining a clearer understanding of the question. My original answer is below.
Here is the OP's array of hashes, modified slightly.
arr = [{ "June" =>{ "A"=>{ 3=>48.4 } } },
{ "January"=>{ "C"=>{ 2=>88.0 } } },
{ "January"=>{ "B"=>{ "D"=>{ 2=>44.0 } } } },
{ "January"=>{ "C"=>{ 2=>10.0 } } },
{ "January"=>{ "C"=>{ 4=>48.8 } } }]
The hash to be constructed appears to be the following.
{ "June" =>[{ "A"=>[{ 3=>48.4 }] }],
"January"=>[{ "B"=>[{ "D"=>[{ 2=>44.0 }] }] }],
"C"=>[{ 2=>98.0, 4=>48.8 }] }] }
Note that 88.0 + 10.0 #=> 98.0 in 2=>98.0.
Observe that all the arrays within arr contain a single element, a hash. That being the case, those arrays serve no useful purpose. I therefore suggest the following hash be constructed instead:
{ "June" =>{ "A"=>{ 3=>48.4 } },
"January"=>{ "B"=>{ "D"=>{ 2=>44.0 } } },
"C"=>{ 2=>98.0, 4=>48.8 } } }
This can be produced with the following recursive method.
def recurse(arr)
arr.map(&:flatten).
group_by(&:first).
each_with_object({}) do |(k,v),h|
o = v.map(&:last)
h.update(k=>o.first.is_a?(Hash) ? recurse(o) : o.sum )
end
end
recurse(arr)
#=> {"June"=>{"A"=>{3=>48.4}},
# "January"=>{"C"=>{2=>98.0, 4=>48.8}, "B"=>{"D"=>{2=>44.0}}}}
(Original answer follows)
Here are two ways to obtain the desired hash. I assume that arr is your array of hashes.
#1 Use the form of Hash::new that takes a block
arr.each_with_object(Hash.new { |h,k| h[k] = [] }) do |g,h|
k, v = g.to_a.first
h[k] << v
end
# => {"June"=>[{"A"=>{3=>48.4}}],
# "January"=>[{"C"=>{2=>88.0}}, {"B"=>{2=>44.0}}, {"C"=>{4=>48.8}}]}
#2 Use Enumerable#group_by
arr.map(&:first).
group_by(&:first).
tap { |h| h.keys.each { |k| h[k] = h[k].map(&:last) } }
# => {"June"=>[{"A"=>{3=>48.4}}],
# "January"=>[{"C"=>{2=>88.0}}, {"B"=>{2=>44.0}}, {"C"=>{4=>48.8}}]}
The steps are as follows.
a = arr.map(&:first)
#=> [["June", {"A"=>{3=>48.4}}], ["January", {"C"=>{2=>88.0}}],
# ["January", {"B"=>{2=>44.0}}], ["January", {"C"=>{4=>48.8}}]]
b = a.group_by(&:first)
#=> {"June"=>[["June", {"A"=>{3=>48.4}}]],
# "January"=>[["January", {"C"=>{2=>88.0}}], ["January", {"B"=>{2=>44.0}}],
# ["January", {"C"=>{4=>48.8}}]]}
c = b.tap { |h| h.keys.each { |k| h[k] = h[k].map(&:last) } }
#=> {"June"=>[{"A"=>{3=>48.4}}],
# "January"=>[{"C"=>{2=>88.0}}, {"B"=>{2=>44.0}}, {"C"=>{=>48.8}}]}
Let me elaborate the last step. Inside tap's block, we compute the following.
h = b
d = h.keys
#=> ["June", "January"]
The first element of d is passed to each's block and the block variable is assigned to that element.
k = d.first
#=> "June"
The block calculation is as follows.
e = h[k]
#=> [["June", {"A"=>{3=>48.4}}]]
f = e.map(&:last)
#=> [{"A"=>{3=>48.4}}]
h[k] = f
#=> [{"A"=>{3=>48.4}}]
b #=> {"June"=>[{"A"=>{3=>48.4}}],
# "January"=>[["January", {"C"=>{2=>88.0}}],
# ["January", {"B"=>{2=>44.0}}],
# ["January", {"C"=>{4=>48.8}}]]}
Next, d[1] ("January") is passed to each's block and similar calculations are performed.
Rather than using Object#tap I could have written
h = arr.map(&:first).
group_by(&:first)
h.keys.each { |k| h[k] = h[k].map(&:last) }
h
tap merely avoids the creation of local variable h and the need to have a final line equal to h.
I'm trying to create JSON files dynamically, where the content of each file belongs only to a certain entity.
I have an array of Ruby objects, where each object represents an entity.
entities = [#<Entity:0x0055f364d9cd78 #name="entity-B", #internal_asn_number=64514, #classification_id="B">,#<Entity:0x0055f364d89070 #name="entity-A", #internal_asn_number=64513, #classification_id="A">]
And I have an array of hashes, which contains several rules for each entity and I would like to write these hashes to JSON files, one file per entity.
I have the following code:
array_hashes = [{"rulename"=>"entity-B_source_fqdn_1", "if"=>{"source.fqdn"=>"mail.tec.dsr.entityB.com"}, "then"=>{"event_description.text"=>"B"}}, {"rulename"=>"entity-B_destination_fqdn_1", "if"=>{"destination.fqdn"=>"mail.tec.dsr.entity_B.com"}, "then"=>{"event_description.text"=>"B"}}, {"rulename"=>"entity-A_source_fqdn_1", "if"=>{"source.fqdn"=>"194-65-57-128.entityA.com"}, "then"=>{"event_description.text"=>"A"}}, {"rulename"=>"entity-A_destination_fqdn_1", "if"=>{"destination.fqdn"=>"194-65-57-128.entityA.com"}, "then"=>{"event_description.text"=>"A"}}]
path = "/home/mf370/Desktop/RubyProject/"
file_name = "_url_fqdn.conf"
entities.each do |entity|
File.open(path + entity.name + file_name, "w") do |f|
array_hashes.each do |rule|
if rule['rulename'].match(entity.name)
f.write(JSON.pretty_generate([rule]))
end
end
end
This code works and creates the files dynamically, however the content of the files is not what I was expecting.
This is the output from the code above:
[
{
"rulename": "entity-A_source_fqdn_1",
"if": {
"source.fqdn": "194-65-57-128.entityA.com"
},
"then": {
"event_description.text": "A"
}
}
][
{
"rulename": "entity-A_destination_fqdn_1",
"if": {
"destination.fqdn": "194-65-57-128.entityA.com"
},
"then": {
"event_description.text": "A"
}
}
]
And this is the output that I was looking for:
[
{
"rulename": "entity-A_source_fqdn_1",
"if": {
"source.fqdn": "194-65-57-128.entityA.com"
},
"then": {
"event_description.text": "A"
}
},
{
"rulename": "entity-A_destination_fqdn_1",
"if": {
"destination.fqdn": "194-65-57-128.entityA.com"
},
"then": {
"event_description.text": "A"
}
}
]
Probably this is very simple to solve, but I ran out of ideas on how to solve this, I'm new to Ruby Programming. Thank you for your help!
The problem is that you pack each rule inside an array :
JSON.pretty_generate([rule])
You should create an array of matching rules, and dump it as JSON :
entities.each do |entity|
File.open(File.join(path, entity.name + file_name), "w") do |f|
matching_rules = array_hashes.select{ |rule| rule['rulename'].match(entity.name) }
f.write(JSON.pretty_generate(matching_rules))
end
end
The Problem is that you wrap each rule in an array.
THis should solve the issue:
json_entities = []
entities.each do |entity|
File.open(path + entity.name + file_name, "w") do |f|
array_hashes.each do |rule|
if rule['rulename'].match(entity.name)
json_entities << rule
end
end
end
end
f.write(JSON.pretty_generate(json_entities))
I have JSON with one object for each site's traffic for 3 dates.
I want to merge these three objects together on "Date". Example below.
I'm using Ruby. What is the easiest way to do this?
Start JSON:
[
{
"Google":[
{
"Date":"2015-01-01",
"Value":100
},
{
"Date":"2015-02-01",
"Value":200
},
{
"Date":"2015-03-01",
"Value":300
}
]
},
{
"Yahoo":[
{
"Date":"2015-01-01",
"Value":1200
},
{
"Date":"2015-02-01",
"Value":1300
},
{
"Date":"2015-03-01",
"Value":1400
}
]
},
{
"Bing":[
{
"Date":"2015-01-01",
"Value":500
},
{
"Date":"2015-02-01",
"Value":600
},
{
"Date":"2015-03-01",
"Value":700
}
]
}
]
End JSON:
[
{
"Date":"2015-01-01",
"Google":100,
"Yahoo":1200,
"Bing":500
},
{
"Date":"2015-01-02",
"Google":200,
"Yahoo":1200,
"Bing":600
},
{
"Date":"2015-01-03",
"Google":300,
"Yahoo":1400,
"Bing":700
}
]
result = array.inject({}) do | a, e |
site, data = e.first
data.each do | x |
a[x[:Date]] ||= {}
a[x[:Date]][site] = x[:Value]
end
a
end
gives you a hash with the dates as keys. This can be transformed to the array by:
result.map { | k, v | v.update(:Date => k) }
Assuming exactly this structure, what might work is
results = []
your_array[0]['Google'].each_with_index do |item, index|
date = item['Date']
provider_values = your_array.inject({}) do |memo, current|
provider = current.keys[0]
value = current[provider][index]['Value']
memo[provider] = value
memo
end
results.push({'Date' => date}.merge(provider_values))
end
I'm currently on Windows so I can't be 100% sure of correctness, however fixing any syntax errors should be easy.