Display dash datatable through callback - reactjs

I want to be able to return a populated dash table based on the results from an input search. I've tried 2 methods so far - returning the entire DashTable in the callback output and returning the columns and data separately in the callback. Both options haven't been working for me. I've included the relevant code for each option and the error message that results from each:
Return the data and columns separately:
#app.callback(
[Output('table', 'data'),
Output('table', 'columns')],
[Input("button", "n_clicks")], state=[State('url', 'value')])
def update_table(n_click:int, url):
if n_click>1:
summary, table = summarizer(url)
columns=[{"name": i, "id": i, "deletable": True, "selectable": True} for i in table.columns]
table = table.to_dict('records')
return table, columns
else:
return [], []
The app.layout contains the following line
html.Div(dt.DataTable(id='table'))
The error message that results from this is:
Objects are not valid as a React child
The second approach was to pass in the entire DataTable through the callback and display it using just the html.Div in the layout like this
#app.callback(
Output('table', 'children'),
[Input("button", "n_clicks")], state=[State('url', 'value')])
def update_table(n_click:int, url):
if n_click>1:
summary, table = summarizer(url)
columns=[{"name": i, "id": i, "deletable": True, "selectable": True} for i in table.columns]
table = table.to_dict('records')
return dt.DataTable(data=table, columns=columns)
else:
return []
html.Div(id='table')
The corresponding error was
[Objects are not valid as a React child][2]
This error is confusing to me since it seems to be regarding the column definition however I can't pass in an array and the documentation asks for a dictionary.
Full code sample:
import dash
import dash_core_components as dcc
import dash_html_components as html
import dash_bootstrap_components as dbc
import dash_table as dt
from dash.dependencies import Input, Output, State
import sd_material_ui
from newspaper import Article
import gensim
from gensim.summarization import summarize
from dash.exceptions import PreventUpdate
from newspaper import fulltext
import requests
import pandas as pd
import yake
import nltk
from newsapi import NewsApiClient
leftSources = ["cnn", "buzzfeed", "the-washington-post", "bbc-news", "vice-news", "newsweek", "techcrunch", "reuters", "politico", "newsweek", "msnbc"]
rightSources = ["fox-news", "national-review", "new-york-magazine", "breitbart-news", "business-insider", "the-wall-street-journal", "bloomberg", "the-washington-times", "the-hill", "the-american-conservative"]
# importing CSS
external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
# similarArticleURL
getSimilarArticlesURL = "https://us-central1-secure-site-266302.cloudfunctions.net/getSimilarArticles?keywords="
getKeywordsURL = "https://us-central1-secure-site-266302.cloudfunctions.net/getKeyword?text="
getArticleTextURL = "https://us-central1-secure-site-266302.cloudfunctions.net/getArticleText?url="
allData = pd.DataFrame()
# instantiating dash application
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
server = app.server # the flask app
# helper functions
def generate_table(dataframe, max_rows=10):
return html.Table([
html.Thead(
html.Tr([html.Th(col) for col in dataframe.columns])
),
html.Tbody([
html.Tr([
html.Td(dataframe.iloc[i][col]) for col in dataframe.columns
]) for i in range(min(len(dataframe), max_rows))
])
])
app.layout = html.Div([
html.Div(html.H3("Brief.Me"), style={'font-weight':'bold','background-color':'darkorange', 'color':'white','text-align':'center'}),
html.Br(),
html.Br(),
dbc.Row([
dbc.Col(dbc.Input(id='url', type='url', size=30, placeholder="Type or copy/paste an URL"), width={'size':6, 'order':1, 'offset':3}),
dbc.Col(dbc.Button("Summarize", id='button', n_clicks=1, color="primary", className="mr-1"), width={'order':2})
]),
html.Br(),
# dbc.Row([
# dbc.Col(dcc.Loading(html.Div(html.Div(id="summary"), style={'font-weight':'bold'})), width={'size':6, 'offset':3})
# ]),
html.Div(id='table')
],
)
def fetch_similar_articles(keyword):
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
newsapi = NewsApiClient(api_key='ce7482cbd40f4d90a8eea404e7702db6')
top_headlines = newsapi.get_top_headlines(q=keyword,
sources='bbc-news,the-wall-street-journal,the-washington-post,fox-news,bloomberg, vice-news, politico, reuters, the-hill',
language='en')
return top_headlines["articles"]
def fetch_article_text(url):
try:
article = Article(url)
article.download()
article.parse()
return article.text
except:
return None
def summarizer(url):
global allData
leftSummaries, rightSummaries = {}, {}
text = fetch_article_text(url)
main_summary = summarize(text)
keywords = extract_keywords(text)
urls = []
rightData, leftData, allData = get_articles_content(keywords)
rightDf, leftDf = pd.DataFrame(rightData), pd.DataFrame(leftData)
allSources = pd.concat([rightDf, leftDf], axis=1)
return main_summary, allData
def get_articles_content(keywords):
'''
This function will return a row of the dataframe where there is a title, source, url and summary.
'''
allResults, leftRows, rightRows = [], [], []
for keyword in keywords:
articleList = fetch_similar_articles(keyword)
for elem in articleList:
source = elem['source']
url = elem['url']
title = elem['title']
text = fetch_article_text(url)
if text is not None and len(text) > 1:
summary = summarize(text)
allResults.append({'title': title, 'url': url,'source': source, 'summary': summary})
if source in leftSources:
leftRows.append(pd.DataFrame({'title': title, 'url': url,'source': source, 'summary': summary}))
elif source in rightSources:
rightRows.append(pd.DataFrame({'title': title, 'url': url, 'source': source, 'summary': summary}))
allResults = pd.DataFrame(allResults)
return leftRows, rightRows, allResults
def extract_keywords_yake(text, phrase_length, num_keywords):
custom_kw_extractor = yake.KeywordExtractor(n=phrase_length, top=num_keywords)
keywords = custom_kw_extractor.extract_keywords(text)
return keywords
def extract_keywords(text):
'''
Returns a list of keywords given the article text.
'''
global getKeywordsURL
getKeywordsURL += text
keywordRes = extract_keywords_yake(text, 2, 5)
keywords = []
for pair in keywordRes:
keywords.append(pair[1])
return keywords
#app.callback( # Output('summary', 'children')
Output('table', 'children'),
[Input("button", "n_clicks")], state=[State('url', 'value')])
def update_table(n_click:int, url):
if n_click>1:
summary, table = summarizer(url)
columns=[{"name": i, "id": i, "deletable": True, "selectable": True} for i in table.columns]
table = table.to_dict('records')
return dt.DataTable(data=table, columns=columns)
else:
return [], []
if __name__ == '__main__':
app.run_server(debug=True, host='0.0.0.0', port=8080)

Related

How to use tensorflow ai trained text model in react

i have created a simple chatbot model in python from a video tutorial.
Now i have read that i can use this model in react with tendorflow.js lib but i cant get it to run. I searched around a while but i cant find a real working example.
1.st the code for creating the model (train)
training.py
import random
import json
import numpy as np
import nltk
nltk.download("punkt")
nltk.download("wordnet")
nltk.download('omw-1.4')
from nltk.stem import WordNetLemmatizer
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras.optimizers import SGD
import tensorflowjs as tfjs
lemmatizer = WordNetLemmatizer()
intents = json.loads(open("./model/Chatbot/intents.json").read())
words = []
classes = []
documents = []
ignore_letters = ["?", "!", ".", ","]
for intent in intents["intents"]:
for pattern in intent["patterns"]:
word_list = nltk.word_tokenize(pattern)
words.extend(word_list)
documents.append((word_list, intent["tag"]))
if intent["tag"] not in classes:
classes.append(intent["tag"])
words = [lemmatizer.lemmatize(word) for word in words if word not in ignore_letters]
words = sorted(set(words))
classes = sorted(set(classes))
pickle.dump(words, open("./model/Chatbot/words.pkl", "wb"))
pickle.dump(classes, open("./model/Chatbot/classes.pkl", "wb"))
training = []
output_empty = [0] * len(classes)
for document in documents:
bag = []
word_patterns = document[0]
word_patterns = [lemmatizer.lemmatize(word.lower()) for word in word_patterns]
for word in words:
bag.append(1) if word in word_patterns else bag.append(0)
output_row = list(output_empty)
output_row[classes.index(document[1])] = 1
training.append([bag, output_row])
random.shuffle(training)
training = np.array(training)
train_x = list(training[:, 0])
train_y = list(training[:, 1])
model = Sequential()
model.add(Dense(128, input_shape=(len(train_x[0]),), activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(64, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(len(train_y[0]), activation="softmax"))
sgd = SGD(learning_rate=0.01, momentum=0.9, nesterov=True)
model.compile(loss="categorical_crossentropy", optimizer=sgd, metrics=["accuracy"])
hist = model.fit(np.array(train_x), np.array(train_y), epochs=1000, batch_size=5, verbose=1)
model.save("./model/Chatbot/chatbotmodel.h5", hist)
tfjs.converters.save_keras_model(model, "./model/Chatbot/")
print("Done")
in the pre-last line the model was exported to model.json and 1 group1-shard1of1.bin file
intents.json (example)
{
"intents": [
{
"tag": "greeting",
"patterns": [
"Hey",
"Hola",
"Hello",
"Hi",
"Ist da jemand?",
"Hallo",
"Guten Tag",
"Hey",
"Moin"
],
"responses": [
"Hallo, schön das du hier bist",
"Schoen dich wiederzusehen",
"Hallo, wie kann ich helfen?"
],
"context_set": "greeting"
}
]
}
in python i can start now chatbot.py which works
import random
import json
import pickle
import numpy as np
import nltk
from nltk.stem import WordNetLemmatizer
from tensorflow import keras
from keras.models import load_model
lemmatizer = WordNetLemmatizer()
intents = json.loads(open("./model/Chatbot/intents.json").read())
words = pickle.load(open("./model/Chatbot/words.pkl", "rb"))
classes = pickle.load(open("./model/Chatbot/classes.pkl", "rb"), fix_imports=True, encoding="ASCII")
model = load_model("./model/Chatbot/chatbotmodel.h5")
context = ""
def clean_up_sentence(sentence):
sentence_words = nltk.word_tokenize(sentence)
sentence_words = [lemmatizer.lemmatize(word) for word in sentence_words]
return sentence_words
def bag_of_words(sentence):
sentence_words = clean_up_sentence(sentence)
bag = [0] * len(words)
for w in sentence_words:
for i, word in enumerate(words):
if word == w:
bag[i] = 1
return np.array(bag)
def predict_class(sentence):
bow = bag_of_words(sentence) # [0 0 0 0 0 0 0 0 0]?
print(np.array([bow]))
res = model.predict(np.array([bow]))[0] # [8.58373418e-02 3.18233818e-02 9.12701711e-02 3.93254980e-02...
print(res)
ERROR_TRESHOLD = 0.25
results = [[i, r] for i, r in enumerate(res) if r > ERROR_TRESHOLD] # Hallo => [[21, 0.35744026]]
results.sort(key=lambda x: x[1], reverse=True) # moin => [[21, 0.35744026]]
return_list = []
for r in results:
return_list.append({"intent": classes[r[0]], "probability": str(r[1])})
return return_list # hallo [{'intent': 'greeting', 'probability': '0.35744026'}]
def get_response(intents_list, intents_json):
tag = intents_list[0]["intent"] # hallo [{'intent': 'greeting', 'probability': '0.35744026'}] ===> 'greeting'
list_of_intents = intents_json["intents"] # ==> alle intents aus datei
print(intents_list)
for i in list_of_intents:
if "context_set" in i:
context = i["context_set"]
print(context)
if i["tag"] == tag:
result = random.choice(i["responses"])
break
return result
print("Go! Bot is running")
while True:
message = input("")
ints = predict_class(message) # # hallo [{'intent': 'greeting', 'probability': '0.35744026'}]
res = get_response(ints, intents)
print(res)
2. Try to get it run in react.
import { useEffect, useState } from 'react';
import * as tf from '#tensorflow/tfjs';
const url = {
model: 'https://example.com/model.json',
};
function App() {
async function loadModel(url) {
try {
let message = "Hallo";
//const inputTensor = tf.tensor([parseInt(message)]);
const model = await tf.loadLayersModel(url.model);
setModel(model);
let result = model.predict(message); // make prediction like in Python
//let bow = bag_of_words(message) // [0 0 0 0 0 0 0 0 0]?
}
catch (err) {
console.log(err);
}
}
useEffect(() => {
tf.ready().then(() => {
loadModel(url)
});
}, [])
}
At this point, the model.json and group1-shard1of1.bin are both imported correct, but when i try to model.predict('hallo') i get the following error:
Error when checking model : the Array of Tensors that you are passing to your model is not the size the the model expected. Expected to see 1 Tensor(s), but instead got 5 Tensors(s).
Maybe u have an idea to solve it? Thanks.

How to increase the timeout on cookiecutter-django

I'm processing some data from the redis cache. But it seems like I cannot process it fast enough to fit within the request timeout. Is there a way to increase the timeout in nginx or django? (I'm not even sure if cookiecutter-django has nginx).
# views.py
from rest_framework import viewsets
from rest_framework.response import Response
from rest_framework.pagination import PageNumberPagination
class SmallResultsSetPagination(PageNumberPagination):
page_size = 5
page_size_query_param = "page_size"
class FooViewSet(viewsets.ModelViewSet):
queryset = Foo.objects.all().order_by("id")
serializer_class = FooSerializer
pagination_class = SmallResultsSetPagination
filterset_fields = ["bar"]
# serializers.py
from rest_framework import serializers
from .models import Foo
class FooSerializer(serializers.ModelSerializer):
id = serializers.IntegerField(read_only=True)
DT_RowId = serializers.SerializerMethodField()
def get_DT_RowId(self, obj):
return obj.id
class Meta:
model = Foo
fields = (
"id",
"DT_RowId",
"name",
"baz",
"api_data",
)
datatables_always_serialize = ("baz", "api_data")
# models.py
import logging
import xml.etree.ElementTree as ElementTree
from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.db import models
from django.utils.functional import cached_property
import requests
from requests.exceptions import ConnectionError, Timeout
logger = logging.getLogger(__name__)
def third_party_api():
bars = cache.get("bars")
if bars:
print("cache hit")
return bars
def bars_to_dict(root):
bars = {}
for bar in root[1]:
bar_name = issuer.tag
entry = {}
for pair in bar:
tag = pair.tag.split("}")[-1]
value = pair.text
entry[tag] = value
key = entry["buzz"].strip().lower()
bars[key] = entry
return bars
try:
r = requests.get(
f"{API}", timeout=5,
)
root = ElementTree.fromstring(r.text)
bars = bars_to_dict(root)
cache.set("bars", bars, 60 * 5)
return bars
except (ConnectionError, Timeout) as e:
if settings.DEBUG:
tree = ElementTree.parse("scripts/bars.xml")
root = tree.getroot()
bars = bars_to_dict(root)
cache.set("bars", bars, 60 * 5)
return bars
else:
return {}
class Foo(models.Model):
baz = models.BooleanField(default=False)
#cached_property
def api_data(foo):
bars = third_party_api()
match = bars.get(foo.id)
if match:
field = match.get("biz", False)
return field == "true"
else:
return False
when I hit the browsable api on staging https://host.com/api/foos/?page_size=7 I get Bad Gateway for page_size values > 7. I'm pretty sure I'm doing too much computation for the default timeout.
The setting is inside settings/base.py
https://github.com/pydanny/cookiecutter-django/blob/8d5542d6754b520e0698286d8a0e6b6fc1257715/%7B%7Bcookiecutter.project_slug%7D%7D/config/settings/base.py#L289
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-time-limit
CELERY_TASK_TIME_LIMIT = 5 * 60
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-soft-time-limit
CELERY_TASK_SOFT_TIME_LIMIT = 60
the units are in seconds.

Creating a dataset

I am trying to extract tweets from a specific hashtag, and save them in csv file. The below code works well, but I would like to split data. How can I split it.
Any advice will be highly appreciated,
Niddal
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import time
import json
import codecs
import sys
ckey = ''
csecret = ''
atoken = ''
asecret = ''
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
class StdOutListener(StreamListener):
def on_data(self, data):
try:
tweet = json.loads(data)['text']
#tweet = data.split(',"text":"')[1].split('","source')[0]
print(tweet.translate(non_bmp_map))
saveThis = str(time.time())+'::'+tweet
SaveFile = codecs.open('d:\\StremHash.csv','a', "utf-8")
SaveFile.write(saveThis)
SaveFile.write('\n')
SaveFile.close()
return True
except BaseException, e:
print ('failed on data,',str(e))
time.sleep(5)
def on_error(self, status):
print(status)
if __name__ == '__main__':
l = StdOutListener()
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, l)
twitterStream.filter(track=[unicode("#عيدكم_مبارك","utf-8")])

Django model import issue via Python interpreter

I was trying to add data to my fields via the Python interpreter(manage.py shell). I can import the other two models (Tag, Startup), but Newslink model throws an error:
...
Traceback (most recent call last):
File "/usr/lib/python3.5/code.py line 91, in runcode
exec(code, self.locals)
File "<console>", line1 in <module>
ImportError: cannot import name 'Newslink'
Typed in Python interpreter:
from organizer.models import Tag
from organizer.models import Startup
from organizer.models import Newslink <--- issue
Here is my /organizer/models.py file:
from django.db import models
class Tag(models.Model):
name = models.CharField(max_length=31,unique = True)
slug = models.SlugField(
max_length=31,
unique = True,
help_text ="A label for URL config.")
def __str__(self):
return self.name.title()
class Meta:
ordering = ['name']
class Startup(models.Model):
name = models.CharField(
max_length=31
,db_index= True)
slug = models.SlugField(
max_length= 31,
unique = True,
help_text = "A label for URL config.")
description = models.TextField()
founded_date = models.DateField('date founded')
contact = models.EmailField()
website = models.URLField(max_length =255)
tags = models.ManyToManyField(Tag)
def __str__(self):
return self.name
class Meta:
ordering = ['name']
get_latest_by = 'founded_date'
class NewsLink(models.Model):
title = models.CharField(max_length=63)
pub_date = models.DateField('date published')
link = models.URLField(max_length=255)
startup = models.ForeignKey(Startup)
def __str__ (self):
return"{}:{}".format (self.startup, self.title)
class Meta:
verbose_name = 'news article'
ordering = ['-pub_date']
get_latest_by = 'pub_date'

Adding methods to GAE database class

I am messing around with GAE. I want to place my database object in one file and call it from another. Here is the DB object:
import webapp2
import os
import jinja2
import json
import logging
import main
from google.appengine.ext import db
class User(db.Model):
user_name = db.StringProperty(required = True)
hashed_password = db.StringProperty(required = True)
email = db.EmailProperty(required = True)
created_dttm = db.DateTimeProperty(auto_now_add = True)
last_modified = db.DateTimeProperty(auto_now = True)
coords = db.GeoPtProperty(required = False)
# def as_dict(self):
# time_fmt = '%c'
# d = {
# 'subject':self.subject,
# 'content':self.content,
# 'created':self.created_dttm.strftime(time_fmt),
# 'last_modified': self.last_modified.strftime(time_fmt)
# }
# return d
def isValueUnique(self,column,value):
result = None
q = User.all()
q.filter(column, value)
result = q.get()
return result
I cannot instantiate the DB because it thinks I'm trying to store data.
I want to call the isValueUnique method from another file like so:
import webapp2
import os
import jinja2
import json
import logging
import main
import database
import validation
from google.appengine.ext import db
class SignUp(main.Handler):
def post(self):
user_username = self.request.get("username")
user_email = self.request.get("email")
user_pass = self.request.get("password")
user_verify = self.request.get("verify")
valid = validation.Valid()
error1=""
error2=""
error3=""
error4=""
q = database.User.all()
q.filter("username =", user_username)
result = q.get()
if result:
error1="Username already taken"
if (not valid.valid_user(user_username)) and (not error1):
error1 = "Enter a valid username"
if not valid.valid_password(user_pass):
error2 = "Enter a valid password"
if not valid.valid_pass_match(user_pass,user_verify):
error3 = "Passwords must match"
# Email Validation
email=valid.valid_email(user_email)
if not email:
error4 = "Invalid email"
email=""
elif not database.User.isValueUnique("email",email):
error4 = "Email already in use, please sign in"
email=""
I get this error:
elif not database.User.isValueUnique("email",email):
TypeError: unbound method isValueUnique() must be called with User instance as first argument (got str instance instead)
I can't instantiate User like I already said. What is the work around here?
database.User.isValueUnique("email",email)
This is attempting to call a method on the database.User class, but isValueUnique is an instance method.
If you decorate isValueUnique with #staticmethod you'll get farther.
Where are you trying to instantiate a User?

Resources