I need to load the sklearn model via pickle file in C. But I do not find how to do that. This is my code and my model:
import pandas
from sklearn import model_selection
from sklearn.linear_model import LogisticRegression
import pickle
url = "https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv"
names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']
dataframe = pandas.read_csv(url, names=names)
array = dataframe.values
X = array[:, 0:8]
Y = array[:, 8]
test_size = 0.33
seed = 7
X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, test_size=test_size, random_state=seed)
# Fit the model on 33%
model = LogisticRegression()
model.fit(X_train, Y_train)
print(model.score(X_test, Y_test))
# save the model to disk
filename = 'finalized_model.sav'
pickle.dump(model, open(filename, 'wb'))
I do not know how to do that. I would be grateful if you could help me, please.
There are always workarounds, One such way could be
Write a python flask based API/microservice over your python code & call it in c using the LibCurl library. I've made an asumption that your test_data will be a single column csv file, All flask apps runs on localhost:5000 by default. I will suggest you to write your own app with proper requests for arguments if your data doesnt't look like this. This piece of code is for reference only.
#python flask code
from flask import Flask
app = Flask(__name__)
#app.route('/')
def your_model():
test_data = pandas.read_csv('test_data.csv')
model = pickle.load("your_model.pkl")
pred_result = model.predict(test_data)
return(pred_result)
app.run()
/* c code*/
CURL *hnd = curl_easy_init();
curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(hnd, CURLOPT_URL, "http://localhost:5000/your_API_name");
struct curl_slist *headers = NULL;
test_data_as_json_string = {""};
headers = curl_slist_append(headers, "any_parameter: value");
headers = curl_slist_append(headers, "content-type: application/json");
curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, test_data_as_json_string);
CURLcode ret = curl_easy_perform(hnd);
This code too is for reference write your own c code or you can copy paste similar kind of code from Postman's code section while running the flask api.
Related
I'm starting a project that must be deployed at the Google App Engine. I read that gluon.tools fetch function must be used instead of urllib.request.
Here I have two defs(), the first one runs ok with urllib, the second one with fetch doesn't run.
It looks as if data parameters has the wrong format.
How must be used this fetch function?
I've been looking for examples along the internet but I don't find anything (except the code of fetchfunction).
# -*- coding: utf-8 -*-
from gluon.tools import fetch
import json
import urllib
def gQLquery1():
data = {'query':'{me{username myMachines{name}}}'}
url = 'https://ecostruxure-machine-advisor.se.app/daas/graphql'
Bearer = "Bearer ngU5TkM5yN2m5VCBeJBQ4F3NGjuvecn8FqK7f7Fc"
data = json.dumps(data).encode('utf-8')
headers={'authorization': Bearer,'Content-Type':'application/json'}
req = urllib.request.Request(url, data=data)
req.add_header('authorization', Bearer)
req.add_header('Content-Type', 'application/json')
response = urllib.request.urlopen(req).read()
decoded = json.loads(response)
out = json.dumps(decoded)
return out
def gQLquery2():
data = {'query':'{me{username myMachines{name}}}'}
url = 'https://ecostruxure-machine-advisor.se.app/daas/graphql'
Bearer = "Bearer ngU5TkM5yN2m5VCBeJBQ4F3NGjuvecn8FqK7f7Fc"
#data = json.dumps(data).encode('utf-8')
headers={'authorization': Bearer,'Content-Type':'application/json'}
#req = urllib.request.Request(url, data=data)
#req.add_header('authorization', Bearer)
#req.add_header('Content-Type', 'application/json')
#response = urllib.request.urlopen(req).read()
response = fetch(url, data , headers )
decoded = json.loads(response)
out = json.dumps(decoded)
return out
ticket returned:
Versión
web2py™ Version 2.21.1-stable+timestamp.2020.11.28.04.10.44
Python Python 3.7.4: D:\Oscar\_Particular\web2py\web2py_no_console.exe (prefix: D:\Oscar\_Particular\web2py)
Traceback (most recent call last):
File "D:\Oscar\_Particular\web2py\gluon\tools.py", line 4639, in fetch
from google.appengine.api import urlfetch
File "D:\Oscar\_Particular\web2py\gluon\custom_import.py", line 85, in custom_importer
return base_importer(pname, globals, locals, fromlist, level)
ModuleNotFoundError: No module named 'applications.hola'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Oscar\_Particular\web2py\gluon\restricted.py", line 219, in restricted
exec(ccode, environment)
File "D:/Oscar/_Particular/web2py/applications/hola/controllers/default.py", line 126, in <module>
File "D:\Oscar\_Particular\web2py\gluon\globals.py", line 430, in <lambda>
self._caller = lambda f: f()
File "D:/Oscar/_Particular/web2py/applications/hola/controllers/default.py", line 94, in gQLquery2
response = fetch(url, data , headers )
File "D:\Oscar\_Particular\web2py\gluon\tools.py", line 4642, in fetch
html = urlopen(req).read()
File "urllib\request.py", line 222, in urlopen
File "urllib\request.py", line 523, in open
File "urllib\request.py", line 1247, in do_request_
TypeError: POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str.
I am developing a web application that takes a word file and performs tokenization.
I noticed that the document is passed correctly from angularJS to Flask, but there is an error that I can't give an explanation:
Traceback (most recent call last):
File "C:\Users\AOUP\MiniAnaconda\lib\site-packages\flask\app.py", line 2446, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\AOUP\MiniAnaconda\lib\site-packages\flask\app.py", line 1951, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\AOUP\MiniAnaconda\lib\site-packages\flask\app.py", line 1820, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\AOUP\MiniAnaconda\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\AOUP\MiniAnaconda\lib\site-packages\flask\app.py", line 1949, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\AOUP\MiniAnaconda\lib\site-packages\flask\app.py", line 1935, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "app.py", line 43, in tokenizer
myDoc = word.Documents.Open(pathToProc, False, False, True) #stackoverflow
File "C:\Users\AOUP\MiniAnaconda\lib\site-packages\win32com\client\dynamic.py", line 527, in __getattr__
raise AttributeError("%s.%s" % (self._username_, attr))
AttributeError: Word.Application.Documents
The document is passed by angularJS with the following code:
var f = document.getElementsByTagName("form")[0].children[1].files[0].name;
if (f != ""){
$http({
url: '/tokenizeDoc',
method: "GET",
params: {doc : f}
});
}
Subsequently it is read by Flask with the following script, and the error falls in the line with the error comment:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import string
import win32com.client
import nltk
import os
from collections import Counter
from pywintypes import com_error
from flask import request, Flask, render_template, jsonify
word = win32com.client.Dispatch("Word.Application")
word.Visible = False
app = Flask(__name__)
#app.route('/')
def landingPage():
return render_template('homepage.html')
#app.route('/tokenizeDoc', methods = ['GET'])
def tokenizer():
if request.method == 'GET':
pathToProc = request.values.get("doc")
sent_tokenizer = nltk.data.load('tokenizers/punkt/italian.pickle')
it_stop_words = nltk.corpus.stopwords.words('italian') + ['\n', '\t', '']
trashes = it_stop_words + list(string.punctuation)
tokensTOT = []
try:
myDoc = word.Documents.Open(pathToProc, False, False, True) #ERROR!!!
sentences = sent_tokenizer.tokenize(word.ActiveDocument.Range().Text)
myDoc.Close()
del myDoc
for sentence in sentences:
tokensTOT = tokensTOT + [t.lower() for t in nltk.word_tokenize(sentence)
if t.lower() not in trashes]
except com_error:
print('IMPOSSIBILE DECIFRARE IL FILE')
return ''
I hope the win32com library is not incompatible with web frameworks and someone can give me an answer.
Many thanks in advance.
use os.path.abspath(pathToProc) instead of pathToProc myDoc = word.Documents.Open(pathToProc, False, False, True) #ERROR!!!
I faced the same problem.
The only solution that I managed to apply is to execute win32com.client.Dispatch individually for each call of the view function.
You also need to execute pythoncom.CoInitialize() for normal working flask multi-threading
import pythoncom
import win32com.client
def get_word_instance():
pythoncom.CoInitialize()
return win32com.client.Dispatch("Word.Application")
#app.route('/tokenizeDoc', methods=['GET'])
def tokenizer():
word = get_word_instance()
word.visible = False
The creation instance of COM-object can be a resource-intensive process. If you need more performance when working with COM objects, then you may need to consider the option of disabling the multithreading of the Flask application.
app.run(threaded=False)
Then you can use your code without any changes
I cant find Alchemy Language API in IBM Watson.
Can I do this with natural-language-understanding service and how?
When I add
from watson_developer_cloud import NaturalLanguageUnderstandingV1
from watson_developer_cloud.natural_language_understanding_v1 \
import Features, EntitiesOptions, KeywordsOptions
It shows some error with combined keyword
# In[]:
import tweepy
import re
import time
import math
import pandas as pd
from watson_developer_cloud import AlchemyLanguageV1
def initAlchemy():
al = AlchemyLanguageV1(api_key='GRYVUMdBbOtJXxNOIs1aopjjaiyOmLG7xJBzkAnvvwLh')
return al
def initTwitterApi():
consumer_key = 'OmK1RrZCVJSRmKxIuQqkBExvw'
consumer_secret = 'VWn6OR4rRgSi7qGnZHCblJMhrSvj1QbJmf0f62uX6ZQWZUUx5q'
access_token = '4852231552-adGooMpTB3EJYPHvs6oGZ40qlo3d2JbVjqUUWkJ'
access_token_secret = 'm9hgeM9p0r1nn8IoQWJYBs5qUQu56XmrAhsDSYKjuiVA4'
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
return api
'''This function is implemented to handle tweepy exception errors
because search is rate limited at 180 queries per 15 minute window by twitter'''
def limit(cursor):
while True:
try:
yield cursor.next()
except tweepy.TweepError as error:
print(repr(error))
print("Twitter Request limit error reached sleeping for 15 minutes")
time.sleep(16*60)
except tweepy.RateLimitError:
print("Rate Limit Error occurred Sleeping for 16 minutes")
time.sleep(16*60)
def retrieveTweets(api, search, lim):
if(lim == ""):
lim = math.inf
else:
lim = int(lim)
text = []
for tweet in limit(tweepy.Cursor(api.search, q=search).items(limit = lim)):
t = re.sub('\s+', ' ', tweet.text)
text.append(t)
data = {"Tweet":text,
"Sentiment":"",
"Score":""}
dataFrame = pd.DataFrame(data, columns=["Tweet","Sentiment","Score"])
return dataFrame
def analyze(al,dataFrame):
sentiment = []
score = []
for i in range(0, dataFrame["Tweet"].__len__()):
res = al.combined(text=dataFrame["Tweet"][i],
extract="doc-sentiment",
sentiment=1)
sentiment.append(res["docSentiment"]["type"])
if(res["docSentiment"]["type"] == "neutral"):
score.append(0)
else:
score.append(res["docSentiment"]["score"])
dataFrame["Sentiment"] = sentiment
dataFrame["Score"] = score
return dataFrame
def main():
#Initialse Twitter Api
api = initTwitterApi()
#Retrieve tweets
dataFrame = retrieveTweets(api,input("Enter the search query (e.g. #hillaryclinton ) : "), input("Enter limit for number of tweets to be searched or else just hit enter : "))
#Initialise IBM Watson Alchemy Language Api
al = initAlchemy()
#Do Document Sentiment analysis
dataFrame = analyze(al, dataFrame)
#Save tweets, sentiment, and score data frame in csv file
dataFrame.to_csv(input("Enter the name of the file (with .csv extension) : "))
if __name__ == '__main__':
main()# -*- coding: utf-8 -*-
The Watson Natural Language Understanding only has a combined call, but since it is the only call, it isn't called combined, its actually analyze. Best place to go for details would be the API documentation - https://www.ibm.com/watson/developercloud/natural-language-understanding/api/v1/?python#post-analyze
I'm trying to embed python in my C application. I download the package in python official website and manage to do a simple Hello World.
Now I want to go deeper and use some libraries of python like numpy, keras, tensorflow...
I'm working with Python 3.5.4, I installed all the needed package on my PC with pip3 :
pip3 install keras
pip3 install tensorflow
...
then I created my script and launch it in python environment, it works fine :
Python:
# Importing the libraries
#
import numpy as np
import pandas as pd
dataset2 = pd.read_csv('I:\RNA\dataset19.csv')
X_test = dataset2.iloc[:, 0:228].values
y_test = dataset2.iloc[:, 228].values
# 2.
import pickle
sc = pickle.load(open('I:\RNA\isVerb_sc', 'rb'))
X_test = sc.transform(X_test)
# 3.
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
classifier = Sequential()
classifier.add(Dense(units = 114, kernel_initializer = 'uniform', activation = 'relu', input_dim = 228))
classifier.add(Dropout(p = 0.3))
classifier.add(Dense(units = 114, kernel_initializer = 'uniform', activation = 'relu'))
classifier.add(Dropout(p = 0.3))
classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
classifier.load_weights('I:\RNA\isVerb_weights.h5')
y_pred = classifier.predict(X_test)
y_pred1 = (y_pred > 0.5)
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred1)
But when I execute the same script in a C environment with embed python it didn't work :
At first, I execute my script directly with PyRun_SimpleFile with no luck, so I sliced it in multiple instructions with PyRun_SimpleString to detect the problem :
C:
result = PyRun_SimpleString("import numpy as np"); // result = 0 (ok)
result = PyRun_SimpleString("import pandas as pd"); // result = 0 (ok)
...
result = PyRun_SimpleString("import pickle"); // result = 0 (ok)
... (all insctruction above works)
result = PyRun_SimpleString("import keras"); // result = -1 !!
... (all under this failed)
but there is not a single stack trace about this error, I tried this but I just got :
"Here's the output: (null)"
My initialization of Python in C seems correct since others libraries import fine :
// Python
wchar_t *stdProgramName = L"I:\\LIBs\\cpython354";
Py_SetProgramName(stdProgramName);
wchar_t *stdPythonHome = L"I:\\LIBs\\cpython354";
Py_SetPythonHome(stdPythonHome);
wchar_t *stdlib = L"I:\\LIBs\\cpython354;I:\\LIBs\\cpython354\\Lib\\python35.zip;I:\\LIBs\\cpython354\\Lib;I:\\LIBs\\cpython354\\DLLs;I:\\LIBs\\cpython354\\Lib\\site-packages";
Py_SetPath(stdlib);
// Initialize Python
Py_Initialize();
When inside a Python cmd, the line import keras take some time (3sec) but works (a warning but I found no harm around it) :
>>> import keras
I:\LIBs\cpython354\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
from ._conv import register_converters as _register_converters
Using TensorFlow backend.
>>>
I'm at loss now, I don't know where to look at since there is no stack trace.
it seems like when you import keras, it executes this line :
sys.stderr.write('Using TensorFlow backend.\n')
or sys.stderr was not defined in python embedded on windows
A simple correction is to define sys.stderr, for example :
import sys
class CatchOutErr:
def __init__(self):
self.value = ''
def write(self, txt):
self.value += txt
catchOutErr = CatchOutErr()
sys.stderr = catchOutErr
Is there any way of using Watsons image classifying abilities to extract information from an image of a document? Rather than simply classifying an image as a, b or c?
Visit here to learn more about watson visual-recognition
Raw code (final code) : #just for reference, read how to create a classifier and implement of your own !
import json
from os.path import join, dirname
from watson_developer_cloud import VisualRecognitionV3
test_url = 'https://url-to-ran-image.jpg'
visual_recognition = VisualRecognitionV3('2017-01-11', api_key='YOUR API KEY')
tree_path = join(dirname(__file__), '../resources/trees.zip')
with open(tree_path, 'rb') as images_file:
tree_results = visual_recognition.classify(images_file=images_file,
threshold=0.1,
classifier_ids=[
'Carsvsmango_1479118188',
'default'])
print(json.dumps(tree_results, indent=2))
url_result = visual_recognition.classify(images_url=test_url)
print(json.dumps(url_result, indent=2))
faces_result = visual_recognition.detect_faces(images_url=test_url)
print(json.dumps(faces_result, indent=2))
print(json.dumps(visual_recognition.list_classifiers(), indent=2))
file_path = join(dirname(__file__), '../some-random-text-image.png')
with open(file_path, 'rb') as image_file:
text_results = visual_recognition.recognize_text(images_file=image_file)
print(json.dumps(text_results, indent=2))
face_path = join(dirname(__file__), '../face.jpg')
with open(face_path, 'rb') as image_file:
face_result = visual_recognition.detect_faces(images_file=image_file)
print(json.dumps(face_result, indent=2))
Diagram that helps in understanding watson's recognition platform well.