search as you type with elasticsearch,angularjs - angularjs

I am working on search as you type functionality with angularjs and elastic search.I am passing the $viewValue to factory written in angular and it fetches data from angular.Please check code below.
services.factory('instantSearch',['$q', 'esFactory', '$location', function($q, elasticsearch, $location){
return{
instantResult : function(term){
var client = elasticsearch({
// host: $location.host() + ':9200'
host: 'localhost:9200'
});
var deferred = $q.defer();
client.search({
"index": 'stocks',
"type": 'stock',
"body": {
"from" : 0, "size" : 20,
"query": {
"bool":{
"should":[
{
"match_phrase":{
"name": term
}
},
{
"match_phrase":{
"symbol": term
}
},
{
"match":{
"industry": term
}
}
]
}
}
}
}).then(function(result) {
var hits = result.hits.hits;
deferred.resolve(hits);
},
function (err) {
console.trace(err.message);
}, deferred.reject);
return deferred.promise;
}
};
}]);
This code is working fine but the problem is that I get result when input matches complete term in elasticsearch index's field.So I want to implement token analyzer which will match token(ngram - 1,2,3) and provide result on typing of each character.
So to add analyzer code we have to add settings in te elasticserach index as below:
"settings": {
"analysis": {
"filter": {
"autocomplete_filter": {
"type": "edge_ngram",
"min_gram": 1,
"max_gram": 20
}
},
"analyzer": {
"autocomplete": {
"type": "custom",
"tokenizer": "standard",
"filter": [
"lowercase",
"autocomplete_filter"
]
}
}
}
}
But I am not getting the way to pass the argument here.Every example I checked shows output with curl command.How can we mix analyzer with the working code above.
Thanks for help.

Have you added the analyzer to the fields name, symbol and industry in your elastic search mapping?
curl -XPUT 'http://localhost:9200/index/type/_mapping?ignore_conflicts=true' -d'
{
"type": {
"properties": {
"name": {
"type": "string",
"analyzer": "autocomplete"
}
}
}
}'
Use ignore_conflicts=true without fail.
If you still face issues, then you might have to create a new index, add analyzer and filter to setting, create the desired mapping and then upload the data again.

Related

elasticsearch aggregates some values in a single field

I have some raw data
{
{
"id":1,
"message":"intercept_log,UDP,0.0.0.0,68,255.255.255.255,67"
},
{
"id":2,
"message":"intercept_log,TCP,172.22.96.4,52085,239.255.255.250,3702,1:"
},
{
"id":3,
"message":"intercept_log,UDP,1.0.0.0,68,255.255.255.255,67"
},
{
"id":4,
"message":"intercept_log,TCP,173.22.96.4,52085,239.255.255.250,3702,1:"
}
}
Demand
I want to group this data by the value of the message part of the message.
Output value like that
{
{
"GroupValue":"TCP",
"DocCount":"2"
},
{
"GroupValue":"UDP",
"DocCount":"2"
}
}
Try
I have tried with these codes but failed
GET systemevent*/_search
{
"size": 0,
"aggs": {
"tags": {
"terms": {
"field": "message.keyword",
"include": " intercept_log[,,](.*?)[,,].*?"
}
}
},
"track_total_hits": true
}
Now I try to use pipelines to meet this need.
"aggs" seems to only group fields.
Does anyone have a better idea?
Link
Terms aggregation
Update
My scene is a little special. I collect logs from many different servers, and then import the logs into es. Therefore, there is a big difference between message fields. If you directly use script statements for grouping statistics, it will result in group failure or inaccurate grouping. I try to filter out some data according to the conditions, and then use script to group the operation code (comment code 1), but this code can't group the correct results.
This is my scene to add:
Our team uses es to analyze the server log, uses rsyslog to forward the data to the server center, and then uses logstash to filter and extract the data to es. At this time, there is a field called message in ES, and the value of message is the detailed log information. At this time, we need to count the data containing some values in the message.
comment code 1
POST systemevent*/_search
{
"size": 0,
"query": {
"bool": {
"must": [
{
"match_phrase": {
"message": {
"query": "intercept_log"
}
}
}
]
}
},
"aggs": {
"protocol": {
"terms": {
"script": "def values = /,/.split(doc['message.keyword'].value); return values.length > 1 ? values[1] : 'N/A'",
"size": 10
}
}
},
"track_total_hits": true
}
comment code 2
POST test2/_search
{
"size": 0,
"aggs": {
"protocol": {
"terms": {
"script": "def values = /.*,.*/.matcher( doc['host.keyword'].value ); if( name.matches() ) {return values.group(1) } else { return 'N/A' }",
"size": 10
}
}
}
}
The easiest way to solve this is by leveraging scripts in the terms aggregation. The script would simply split on commas and take the second value.
POST systemevent*/_search
{
"size": 0,
"aggs": {
"protocol": {
"terms": {
"script": "def values = /,/.split(doc['message.keyword'].value); return values.length > 1 ? values[1] : 'N/A';",
"size": 10
}
}
}
}
Use Regex
POST test2/_search
{
"size": 0,
"aggs": {
"protocol": {
"terms": {
"script": "def m = /.*proto='(.*?)'./.matcher(doc['message.keyword'].value ); if( m.matches() ) { return m.group(1) } else { return 'N/A' }"
}
}
}
}
The results would look like
"buckets" : [
{
"key" : "TCP",
"doc_count" : 2
},
{
"key" : "UDP",
"doc_count" : 2
}
]
A better and more efficient way would be to split the message field into new fields using an ingest pipeline or Logstash.

Multiple match_phrase conditions with another bool in a single ElasticSearch query?

I am trying to conduct an Elasticsearch query that searched a text field ("body") and returns items that match at least one of two multi-word phrases I provide (ie: "stack overflow" OR "the stackoverflow"). I would also like the query to only provide results that occur after a given timestamp, with the results ordered by time.
My current solution is below. I believe the MUST is working correctly (gte a timestamp), but the BOOL + SHOULD with two match_phrases is not correct. I am getting the following error:
Unexpected character ('{' (code 123)): was expecting double-quote to start field name
Which I think is because I have two match_phrases in there?
This is the ES mapping and the details of the ES API I am using details are here.
{"query":
{"bool":
{"should":
[{"match_phrase":
{"body":"a+phrase"}
},
{"match_phrase":
{"body":"another+phrase"}
}
]
},
{"bool":
{"must":
[{"range":
{"created_at:
{"gte":"thispage"}
}
}
]}
}
},"size":10000,
"sort":"created_at"
}
I think you were just missing a single " after created_at.
{
"query": {
"bool": {
"must": [
{
"range": {
"created_at": {
"gte": "1534004694"
}
}
},
{
"bool": {
"should": [
{
"match_phrase": {
"body": "a+phrase"
}
},
{
"match_phrase": {
"body": "another+phrase"
}
}
]
}
}
]
}
},
"size": 10,
"sort": "created_at"
}
Also, you are allowed to have both must and should as properties of a bool object, so this is also worth trying.
{
"query": {
"bool": {
"must": {
"range": {
"created_at": {
"gte": "1534004694"
}
}
},
"should": [
{
"match_phrase": {
"body": "a+phrase"
}
},
{
"match_phrase": {
"body": "another+phrase"
}
}
]
}
},
"size": 10,
"sort": "created_at"
}
On a side note, Postman or any JSON formatter/validator would really help in determining where the error is.

Remove elements/objects From Array in ElasticSearch Followed by Matching Query

I'm having issues trying to remove elements/objects from an array in elasticsearch.
This is the mapping for the index:
{
"example1": {
"mappings": {
"doc": {
"properties": {
"locations": {
"type": "geo_point"
},
"postDate": {
"type": "date"
},
"status": {
"type": "long"
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
And this is an example document.
{
"_index": "example1",
"_type": "doc",
"_id": "8036",
"_score": 1,
"_source": {
"user": "kimchy8036",
"postDate": "2009-11-15T13:12:00",
"locations": [
[
72.79887719999999,
21.193036000000003
],
[
-1.8262150000000001,
51.178881999999994
]
]
}
}
Using the query below, I can add multiple locations.
POST /example1/_update_by_query
{
"query": {
"match": {
"_id": "3"
}
},
"script": {
"lang": "painless",
"inline": "ctx._source.locations.add(params.newsupp)",
"params": {
"newsupp": [
-74.00,
41.12121
]
}
}
}
But I'm not able to remove array objects from locations. I have tried the query below but it's not working.
POST example1/doc/3/_update
{
"script": {
"lang": "painless",
"inline": "ctx._source.locations.remove(params.tag)",
"params": {
"tag": [
-74.00,
41.12121
]
}
}
}
Kindly let me know where i am doing wrong here. I am using elastic version 5.5.2
In painless scripts, Array.remove() method removes by index, not by value.
Here's a working example that removes array elements by value in Elasticsearch script:
POST objects/_update_by_query
{
"query": {
... // use regular ES query to remove only in relevant documents
},
"script": {
"source": """
if (ctx._source[params.array_attribute] != null) {
for (int i=ctx._source[params.array_attribute].length-1; i>=0; i--) {
if (ctx._source[params.array_attribute][i] == params.value_to_remove) {
ctx._source[params.array_attribute].remove(i);
}
}
}
""",
"params": {
"array_attribute": "<NAME_OF_ARRAY_PROPERTY_TO_REMOVE_VALUE_IN>",
"value_to_remove": "<VALUE_TO_REMOVE_FROM_ARRAY>",
}
}
}
You might want to simplify script, if your script shall only remove values from one specific array attribute. For example, removing "green" from document's .color_list array:
_doc/001 = {
"color_list": ["red", "blue", "green"]
}
Script to remove "green":
POST objects/_update_by_query
{
"query": {
... // use regular ES query to remove only in relevant documents
},
"script": {
"source": """
for (int i=ctx._source.color_list.length-1; i>=0; i--) {
if (ctx._source.color_list[i] == params.color_to_remove) {
ctx._source.color_list.remove(i);
}
}
""",
"params": {
"color_to_remove": "green"
}
}
}
Unlike add(), remove() takes the index of the element and remove it.
Your ctx._source.locations in painless is an ArrayList. It has List's remove() method:
E remove(int index)
Removes the element at the specified position in this list (optional operation). ...
See Painless API - List for other methods.
See this answer for example code.
"script" : {
"lang":"painless",
"inline":"ctx._source.locations.remove(params.tag)",
"params":{
"tag":indexToRemove
}
}
If with ctx._source.locations.add(elt) You add the element, with ctx._source.locations.remove(indexToRemove), you remove by the index of element in the array.

dynamically add suggestion chips on Api.ai for Actions on google

I want to add suggestions for the user in my Google Assistant Bot. I am using API.ai for bot development and using fulfilment, I am communicating with my backend for data.
I am not able to send suggestions using suggestions chips to my bot.
I have followed as answered here Webhook response with "suggestion chips"
as well as the document at https://developers.google.com/actions/assistant/responses#json.
But still, I only see simple text response at my bot on device as well as on simulator.
I also checked at https://discuss.api.ai/t/google-assistant-rich-message-responses/5134/19. But didn't find any way to switch to V1 or V2. The sample format also didn't work!
Here are my 2 JSONs:
at API.ai
"fulfillment": {
"speech": "want to proceed further?",
"messages": [
{
"type": 0,
"speech": "want to proceed further?"
}
],
"data": {
"google": {
"conversationToken": "[\"AS-PER-JSON-FROM-SIMULATOR\"]",
"expectedInputs": [
{
"inputPrompt": {
"richInitialPrompt": {
"items": [
{
"simpleResponse": {
"textToSpeech": "want to proceed further?",
"displayText": "want to proceed further?"
}
}
],
"suggestions": [
{
"title": "Yes"
},
{
"title": "No"
}
]
}
}
}
]
}
}
},
at action on Google
"expectUserResponse": true,
"expectedInputs": [
{
"inputPrompt": {
"richInitialPrompt": {
"items": [
{
"simpleResponse": {
"textToSpeech": "want to proceed?"
}
}
]
},
"noMatchPrompts": [],
"noInputPrompts": []
},
"possibleIntents": [
{
"intent": "assistant.intent.action.TEXT"
}
],
"speechBiasingHints": [
"$subject",
"$answer"
]
}
]
python server
return = '{"speech":"want to proceed?", "data": {"google":{"expectedInputs":[{"inputPrompt":{"richInitialPrompt":{"items":[{"simpleResponse":{"textToSpeech":"want to proceed?","displayText":"want to proceed?"}}],"suggestions":[{"title":"Yes"},{"title":"No"}]}}}]}}}'
Your JSON is wrong, remove the quotation mark before the data object:
"data" : { ... }
instead of
"data" : "{ ... }"
So basically, you're sending a string containing the object instead of a JSON object.
Solved using format as explained here https://developers.google.com/actions/apiai/webhook
Add 'expectUserResponse' into data -> google
'expectUserResponse': true,
'isSsml': false,

How to remove a property before save

I'm using Angular to build my frontend and loopback at the backend, My Model has a relation HasManyThrough
// person.json
{
"name": "Person",
...,
"relations": {
"contacts": {
"type": "hasMany",
"model": "Person",
"foreignKey": "fromId",
"through": "PersonConnect"
}
}
}
// person-connect.json
{
"name": "PersonConnect",
"base": "PersistedModel",
"properties": ...,
"relations": {
"from": {
"type": "belongsTo",
"model": "Person",
"foreignKey": "fromId"
},
"to": {
"type": "belongsTo",
"model": "Person",
"foreignKey": "toId"
}
}
}
If I try with the explorer I can build a new relation between two people using
PUT /api/Person/:id/contacts/:fk
Where id, the fromId and fk is the toId, the problem is Angular SDK generate services is sending also the body parameters id and fk, this make problems because the set the PersonConnect.id equal to Person.fromId and also appends a extra value fk
{
"_id" : ObjectId("55f0915f19c46e06675d056e"),
...
"fromId" : ObjectId("55f0915f19c46e06675d056e"),
"toId" : ObjectId("55f09b4d4d06f8c872e43c84"),
"fk" : "55f09b4d4d06f8c872e43c84"
}
To fixed I write the following
// person-connect.js
var _ = require('lodash');
module.exports = function (PersonConnect) {
PersonConnect.observe('before save', function (ctx, next) {
if (ctx.instance) {
ctx.instance = _.omit(ctx.instance, ['id', 'fk']);
}
next();
});
};
Without success, the id, and fk values still are using the send values, I set to null and works but I get something like
{
"_id" : ObjectId("55f18c67bbfa11053b36cafc"),
...
"fromId" : ObjectId("55f0915f19c46e06675d056e"),
"toId" : ObjectId("55f09b4d4d06f8c872e43c84"),
"fk" : null
}
How I can delete properties before store a model in loopback?
You can try using unsetAttribute instead:
ctx.instance.unsetAttribute('unwantedField');

Resources