Display dash datatable through callback - reactjs
I want to be able to return a populated dash table based on the results from an input search. I've tried 2 methods so far - returning the entire DashTable in the callback output and returning the columns and data separately in the callback. Both options haven't been working for me. I've included the relevant code for each option and the error message that results from each:
Return the data and columns separately:
#app.callback(
[Output('table', 'data'),
Output('table', 'columns')],
[Input("button", "n_clicks")], state=[State('url', 'value')])
def update_table(n_click:int, url):
if n_click>1:
summary, table = summarizer(url)
columns=[{"name": i, "id": i, "deletable": True, "selectable": True} for i in table.columns]
table = table.to_dict('records')
return table, columns
else:
return [], []
The app.layout contains the following line
html.Div(dt.DataTable(id='table'))
The error message that results from this is:
Objects are not valid as a React child
The second approach was to pass in the entire DataTable through the callback and display it using just the html.Div in the layout like this
#app.callback(
Output('table', 'children'),
[Input("button", "n_clicks")], state=[State('url', 'value')])
def update_table(n_click:int, url):
if n_click>1:
summary, table = summarizer(url)
columns=[{"name": i, "id": i, "deletable": True, "selectable": True} for i in table.columns]
table = table.to_dict('records')
return dt.DataTable(data=table, columns=columns)
else:
return []
html.Div(id='table')
The corresponding error was
[Objects are not valid as a React child][2]
This error is confusing to me since it seems to be regarding the column definition however I can't pass in an array and the documentation asks for a dictionary.
Full code sample:
import dash
import dash_core_components as dcc
import dash_html_components as html
import dash_bootstrap_components as dbc
import dash_table as dt
from dash.dependencies import Input, Output, State
import sd_material_ui
from newspaper import Article
import gensim
from gensim.summarization import summarize
from dash.exceptions import PreventUpdate
from newspaper import fulltext
import requests
import pandas as pd
import yake
import nltk
from newsapi import NewsApiClient
leftSources = ["cnn", "buzzfeed", "the-washington-post", "bbc-news", "vice-news", "newsweek", "techcrunch", "reuters", "politico", "newsweek", "msnbc"]
rightSources = ["fox-news", "national-review", "new-york-magazine", "breitbart-news", "business-insider", "the-wall-street-journal", "bloomberg", "the-washington-times", "the-hill", "the-american-conservative"]
# importing CSS
external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
# similarArticleURL
getSimilarArticlesURL = "https://us-central1-secure-site-266302.cloudfunctions.net/getSimilarArticles?keywords="
getKeywordsURL = "https://us-central1-secure-site-266302.cloudfunctions.net/getKeyword?text="
getArticleTextURL = "https://us-central1-secure-site-266302.cloudfunctions.net/getArticleText?url="
allData = pd.DataFrame()
# instantiating dash application
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
server = app.server # the flask app
# helper functions
def generate_table(dataframe, max_rows=10):
return html.Table([
html.Thead(
html.Tr([html.Th(col) for col in dataframe.columns])
),
html.Tbody([
html.Tr([
html.Td(dataframe.iloc[i][col]) for col in dataframe.columns
]) for i in range(min(len(dataframe), max_rows))
])
])
app.layout = html.Div([
html.Div(html.H3("Brief.Me"), style={'font-weight':'bold','background-color':'darkorange', 'color':'white','text-align':'center'}),
html.Br(),
html.Br(),
dbc.Row([
dbc.Col(dbc.Input(id='url', type='url', size=30, placeholder="Type or copy/paste an URL"), width={'size':6, 'order':1, 'offset':3}),
dbc.Col(dbc.Button("Summarize", id='button', n_clicks=1, color="primary", className="mr-1"), width={'order':2})
]),
html.Br(),
# dbc.Row([
# dbc.Col(dcc.Loading(html.Div(html.Div(id="summary"), style={'font-weight':'bold'})), width={'size':6, 'offset':3})
# ]),
html.Div(id='table')
],
)
def fetch_similar_articles(keyword):
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
newsapi = NewsApiClient(api_key='ce7482cbd40f4d90a8eea404e7702db6')
top_headlines = newsapi.get_top_headlines(q=keyword,
sources='bbc-news,the-wall-street-journal,the-washington-post,fox-news,bloomberg, vice-news, politico, reuters, the-hill',
language='en')
return top_headlines["articles"]
def fetch_article_text(url):
try:
article = Article(url)
article.download()
article.parse()
return article.text
except:
return None
def summarizer(url):
global allData
leftSummaries, rightSummaries = {}, {}
text = fetch_article_text(url)
main_summary = summarize(text)
keywords = extract_keywords(text)
urls = []
rightData, leftData, allData = get_articles_content(keywords)
rightDf, leftDf = pd.DataFrame(rightData), pd.DataFrame(leftData)
allSources = pd.concat([rightDf, leftDf], axis=1)
return main_summary, allData
def get_articles_content(keywords):
'''
This function will return a row of the dataframe where there is a title, source, url and summary.
'''
allResults, leftRows, rightRows = [], [], []
for keyword in keywords:
articleList = fetch_similar_articles(keyword)
for elem in articleList:
source = elem['source']
url = elem['url']
title = elem['title']
text = fetch_article_text(url)
if text is not None and len(text) > 1:
summary = summarize(text)
allResults.append({'title': title, 'url': url,'source': source, 'summary': summary})
if source in leftSources:
leftRows.append(pd.DataFrame({'title': title, 'url': url,'source': source, 'summary': summary}))
elif source in rightSources:
rightRows.append(pd.DataFrame({'title': title, 'url': url, 'source': source, 'summary': summary}))
allResults = pd.DataFrame(allResults)
return leftRows, rightRows, allResults
def extract_keywords_yake(text, phrase_length, num_keywords):
custom_kw_extractor = yake.KeywordExtractor(n=phrase_length, top=num_keywords)
keywords = custom_kw_extractor.extract_keywords(text)
return keywords
def extract_keywords(text):
'''
Returns a list of keywords given the article text.
'''
global getKeywordsURL
getKeywordsURL += text
keywordRes = extract_keywords_yake(text, 2, 5)
keywords = []
for pair in keywordRes:
keywords.append(pair[1])
return keywords
#app.callback( # Output('summary', 'children')
Output('table', 'children'),
[Input("button", "n_clicks")], state=[State('url', 'value')])
def update_table(n_click:int, url):
if n_click>1:
summary, table = summarizer(url)
columns=[{"name": i, "id": i, "deletable": True, "selectable": True} for i in table.columns]
table = table.to_dict('records')
return dt.DataTable(data=table, columns=columns)
else:
return [], []
if __name__ == '__main__':
app.run_server(debug=True, host='0.0.0.0', port=8080)
Related
How to use tensorflow ai trained text model in react
i have created a simple chatbot model in python from a video tutorial. Now i have read that i can use this model in react with tendorflow.js lib but i cant get it to run. I searched around a while but i cant find a real working example. 1.st the code for creating the model (train) training.py import random import json import numpy as np import nltk nltk.download("punkt") nltk.download("wordnet") nltk.download('omw-1.4') from nltk.stem import WordNetLemmatizer from keras.models import Sequential from keras.layers import Dense, Activation, Dropout from keras.optimizers import SGD import tensorflowjs as tfjs lemmatizer = WordNetLemmatizer() intents = json.loads(open("./model/Chatbot/intents.json").read()) words = [] classes = [] documents = [] ignore_letters = ["?", "!", ".", ","] for intent in intents["intents"]: for pattern in intent["patterns"]: word_list = nltk.word_tokenize(pattern) words.extend(word_list) documents.append((word_list, intent["tag"])) if intent["tag"] not in classes: classes.append(intent["tag"]) words = [lemmatizer.lemmatize(word) for word in words if word not in ignore_letters] words = sorted(set(words)) classes = sorted(set(classes)) pickle.dump(words, open("./model/Chatbot/words.pkl", "wb")) pickle.dump(classes, open("./model/Chatbot/classes.pkl", "wb")) training = [] output_empty = [0] * len(classes) for document in documents: bag = [] word_patterns = document[0] word_patterns = [lemmatizer.lemmatize(word.lower()) for word in word_patterns] for word in words: bag.append(1) if word in word_patterns else bag.append(0) output_row = list(output_empty) output_row[classes.index(document[1])] = 1 training.append([bag, output_row]) random.shuffle(training) training = np.array(training) train_x = list(training[:, 0]) train_y = list(training[:, 1]) model = Sequential() model.add(Dense(128, input_shape=(len(train_x[0]),), activation="relu")) model.add(Dropout(0.5)) model.add(Dense(64, activation="relu")) model.add(Dropout(0.5)) model.add(Dense(len(train_y[0]), activation="softmax")) sgd = SGD(learning_rate=0.01, momentum=0.9, nesterov=True) model.compile(loss="categorical_crossentropy", optimizer=sgd, metrics=["accuracy"]) hist = model.fit(np.array(train_x), np.array(train_y), epochs=1000, batch_size=5, verbose=1) model.save("./model/Chatbot/chatbotmodel.h5", hist) tfjs.converters.save_keras_model(model, "./model/Chatbot/") print("Done") in the pre-last line the model was exported to model.json and 1 group1-shard1of1.bin file intents.json (example) { "intents": [ { "tag": "greeting", "patterns": [ "Hey", "Hola", "Hello", "Hi", "Ist da jemand?", "Hallo", "Guten Tag", "Hey", "Moin" ], "responses": [ "Hallo, schön das du hier bist", "Schoen dich wiederzusehen", "Hallo, wie kann ich helfen?" ], "context_set": "greeting" } ] } in python i can start now chatbot.py which works import random import json import pickle import numpy as np import nltk from nltk.stem import WordNetLemmatizer from tensorflow import keras from keras.models import load_model lemmatizer = WordNetLemmatizer() intents = json.loads(open("./model/Chatbot/intents.json").read()) words = pickle.load(open("./model/Chatbot/words.pkl", "rb")) classes = pickle.load(open("./model/Chatbot/classes.pkl", "rb"), fix_imports=True, encoding="ASCII") model = load_model("./model/Chatbot/chatbotmodel.h5") context = "" def clean_up_sentence(sentence): sentence_words = nltk.word_tokenize(sentence) sentence_words = [lemmatizer.lemmatize(word) for word in sentence_words] return sentence_words def bag_of_words(sentence): sentence_words = clean_up_sentence(sentence) bag = [0] * len(words) for w in sentence_words: for i, word in enumerate(words): if word == w: bag[i] = 1 return np.array(bag) def predict_class(sentence): bow = bag_of_words(sentence) # [0 0 0 0 0 0 0 0 0]? print(np.array([bow])) res = model.predict(np.array([bow]))[0] # [8.58373418e-02 3.18233818e-02 9.12701711e-02 3.93254980e-02... print(res) ERROR_TRESHOLD = 0.25 results = [[i, r] for i, r in enumerate(res) if r > ERROR_TRESHOLD] # Hallo => [[21, 0.35744026]] results.sort(key=lambda x: x[1], reverse=True) # moin => [[21, 0.35744026]] return_list = [] for r in results: return_list.append({"intent": classes[r[0]], "probability": str(r[1])}) return return_list # hallo [{'intent': 'greeting', 'probability': '0.35744026'}] def get_response(intents_list, intents_json): tag = intents_list[0]["intent"] # hallo [{'intent': 'greeting', 'probability': '0.35744026'}] ===> 'greeting' list_of_intents = intents_json["intents"] # ==> alle intents aus datei print(intents_list) for i in list_of_intents: if "context_set" in i: context = i["context_set"] print(context) if i["tag"] == tag: result = random.choice(i["responses"]) break return result print("Go! Bot is running") while True: message = input("") ints = predict_class(message) # # hallo [{'intent': 'greeting', 'probability': '0.35744026'}] res = get_response(ints, intents) print(res) 2. Try to get it run in react. import { useEffect, useState } from 'react'; import * as tf from '#tensorflow/tfjs'; const url = { model: 'https://example.com/model.json', }; function App() { async function loadModel(url) { try { let message = "Hallo"; //const inputTensor = tf.tensor([parseInt(message)]); const model = await tf.loadLayersModel(url.model); setModel(model); let result = model.predict(message); // make prediction like in Python //let bow = bag_of_words(message) // [0 0 0 0 0 0 0 0 0]? } catch (err) { console.log(err); } } useEffect(() => { tf.ready().then(() => { loadModel(url) }); }, []) } At this point, the model.json and group1-shard1of1.bin are both imported correct, but when i try to model.predict('hallo') i get the following error: Error when checking model : the Array of Tensors that you are passing to your model is not the size the the model expected. Expected to see 1 Tensor(s), but instead got 5 Tensors(s). Maybe u have an idea to solve it? Thanks.
How to increase the timeout on cookiecutter-django
I'm processing some data from the redis cache. But it seems like I cannot process it fast enough to fit within the request timeout. Is there a way to increase the timeout in nginx or django? (I'm not even sure if cookiecutter-django has nginx). # views.py from rest_framework import viewsets from rest_framework.response import Response from rest_framework.pagination import PageNumberPagination class SmallResultsSetPagination(PageNumberPagination): page_size = 5 page_size_query_param = "page_size" class FooViewSet(viewsets.ModelViewSet): queryset = Foo.objects.all().order_by("id") serializer_class = FooSerializer pagination_class = SmallResultsSetPagination filterset_fields = ["bar"] # serializers.py from rest_framework import serializers from .models import Foo class FooSerializer(serializers.ModelSerializer): id = serializers.IntegerField(read_only=True) DT_RowId = serializers.SerializerMethodField() def get_DT_RowId(self, obj): return obj.id class Meta: model = Foo fields = ( "id", "DT_RowId", "name", "baz", "api_data", ) datatables_always_serialize = ("baz", "api_data") # models.py import logging import xml.etree.ElementTree as ElementTree from django.conf import settings from django.contrib.auth import get_user_model from django.core.cache import cache from django.db import models from django.utils.functional import cached_property import requests from requests.exceptions import ConnectionError, Timeout logger = logging.getLogger(__name__) def third_party_api(): bars = cache.get("bars") if bars: print("cache hit") return bars def bars_to_dict(root): bars = {} for bar in root[1]: bar_name = issuer.tag entry = {} for pair in bar: tag = pair.tag.split("}")[-1] value = pair.text entry[tag] = value key = entry["buzz"].strip().lower() bars[key] = entry return bars try: r = requests.get( f"{API}", timeout=5, ) root = ElementTree.fromstring(r.text) bars = bars_to_dict(root) cache.set("bars", bars, 60 * 5) return bars except (ConnectionError, Timeout) as e: if settings.DEBUG: tree = ElementTree.parse("scripts/bars.xml") root = tree.getroot() bars = bars_to_dict(root) cache.set("bars", bars, 60 * 5) return bars else: return {} class Foo(models.Model): baz = models.BooleanField(default=False) #cached_property def api_data(foo): bars = third_party_api() match = bars.get(foo.id) if match: field = match.get("biz", False) return field == "true" else: return False when I hit the browsable api on staging https://host.com/api/foos/?page_size=7 I get Bad Gateway for page_size values > 7. I'm pretty sure I'm doing too much computation for the default timeout.
The setting is inside settings/base.py https://github.com/pydanny/cookiecutter-django/blob/8d5542d6754b520e0698286d8a0e6b6fc1257715/%7B%7Bcookiecutter.project_slug%7D%7D/config/settings/base.py#L289 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-time-limit CELERY_TASK_TIME_LIMIT = 5 * 60 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-soft-time-limit CELERY_TASK_SOFT_TIME_LIMIT = 60 the units are in seconds.
Creating a dataset
I am trying to extract tweets from a specific hashtag, and save them in csv file. The below code works well, but I would like to split data. How can I split it. Any advice will be highly appreciated, Niddal # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function from tweepy import Stream from tweepy import OAuthHandler from tweepy.streaming import StreamListener import time import json import codecs import sys ckey = '' csecret = '' atoken = '' asecret = '' non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd) class StdOutListener(StreamListener): def on_data(self, data): try: tweet = json.loads(data)['text'] #tweet = data.split(',"text":"')[1].split('","source')[0] print(tweet.translate(non_bmp_map)) saveThis = str(time.time())+'::'+tweet SaveFile = codecs.open('d:\\StremHash.csv','a', "utf-8") SaveFile.write(saveThis) SaveFile.write('\n') SaveFile.close() return True except BaseException, e: print ('failed on data,',str(e)) time.sleep(5) def on_error(self, status): print(status) if __name__ == '__main__': l = StdOutListener() auth = OAuthHandler(ckey, csecret) auth.set_access_token(atoken, asecret) twitterStream = Stream(auth, l) twitterStream.filter(track=[unicode("#عيدكم_مبارك","utf-8")])
Django model import issue via Python interpreter
I was trying to add data to my fields via the Python interpreter(manage.py shell). I can import the other two models (Tag, Startup), but Newslink model throws an error: ... Traceback (most recent call last): File "/usr/lib/python3.5/code.py line 91, in runcode exec(code, self.locals) File "<console>", line1 in <module> ImportError: cannot import name 'Newslink' Typed in Python interpreter: from organizer.models import Tag from organizer.models import Startup from organizer.models import Newslink <--- issue Here is my /organizer/models.py file: from django.db import models class Tag(models.Model): name = models.CharField(max_length=31,unique = True) slug = models.SlugField( max_length=31, unique = True, help_text ="A label for URL config.") def __str__(self): return self.name.title() class Meta: ordering = ['name'] class Startup(models.Model): name = models.CharField( max_length=31 ,db_index= True) slug = models.SlugField( max_length= 31, unique = True, help_text = "A label for URL config.") description = models.TextField() founded_date = models.DateField('date founded') contact = models.EmailField() website = models.URLField(max_length =255) tags = models.ManyToManyField(Tag) def __str__(self): return self.name class Meta: ordering = ['name'] get_latest_by = 'founded_date' class NewsLink(models.Model): title = models.CharField(max_length=63) pub_date = models.DateField('date published') link = models.URLField(max_length=255) startup = models.ForeignKey(Startup) def __str__ (self): return"{}:{}".format (self.startup, self.title) class Meta: verbose_name = 'news article' ordering = ['-pub_date'] get_latest_by = 'pub_date'
Adding methods to GAE database class
I am messing around with GAE. I want to place my database object in one file and call it from another. Here is the DB object: import webapp2 import os import jinja2 import json import logging import main from google.appengine.ext import db class User(db.Model): user_name = db.StringProperty(required = True) hashed_password = db.StringProperty(required = True) email = db.EmailProperty(required = True) created_dttm = db.DateTimeProperty(auto_now_add = True) last_modified = db.DateTimeProperty(auto_now = True) coords = db.GeoPtProperty(required = False) # def as_dict(self): # time_fmt = '%c' # d = { # 'subject':self.subject, # 'content':self.content, # 'created':self.created_dttm.strftime(time_fmt), # 'last_modified': self.last_modified.strftime(time_fmt) # } # return d def isValueUnique(self,column,value): result = None q = User.all() q.filter(column, value) result = q.get() return result I cannot instantiate the DB because it thinks I'm trying to store data. I want to call the isValueUnique method from another file like so: import webapp2 import os import jinja2 import json import logging import main import database import validation from google.appengine.ext import db class SignUp(main.Handler): def post(self): user_username = self.request.get("username") user_email = self.request.get("email") user_pass = self.request.get("password") user_verify = self.request.get("verify") valid = validation.Valid() error1="" error2="" error3="" error4="" q = database.User.all() q.filter("username =", user_username) result = q.get() if result: error1="Username already taken" if (not valid.valid_user(user_username)) and (not error1): error1 = "Enter a valid username" if not valid.valid_password(user_pass): error2 = "Enter a valid password" if not valid.valid_pass_match(user_pass,user_verify): error3 = "Passwords must match" # Email Validation email=valid.valid_email(user_email) if not email: error4 = "Invalid email" email="" elif not database.User.isValueUnique("email",email): error4 = "Email already in use, please sign in" email="" I get this error: elif not database.User.isValueUnique("email",email): TypeError: unbound method isValueUnique() must be called with User instance as first argument (got str instance instead) I can't instantiate User like I already said. What is the work around here?
database.User.isValueUnique("email",email) This is attempting to call a method on the database.User class, but isValueUnique is an instance method. If you decorate isValueUnique with #staticmethod you'll get farther. Where are you trying to instantiate a User?