I am developing a flask web application currently. However, I do not know how I can get the uploaded images from the user into the SQLite database and retrieve it for later use (such as display it on the homepage and other pages).
I am very new to web development so I am not familar with many programming languages yet. I have seen other pages talking about the use of php, may I know if that is really needed? Are there other ways of doing so in flask?
Would appreciate it if someone is able to guide me.
Thank you!
You can base64 encode an image and use a data: url
<img src="data:image/png;base64, iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==" alt="Red dot" />
if you simple paste
data:image/png;base64, iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==
in your address bar, you will see a small red dot
In an app this looks like this:
# FLASK_APP=mini_app flask run
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
class User(db.Model):
__tablename__ = "Users"
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String)
image = db.Column(db.String)
def create_app(config_filename=None, host="localhost"):
app = Flask("demo_app")
app.config.update(SQLALCHEMY_DATABASE_URI="sqlite://")
db.init_app(app)
with app.app_context():
db.create_all()
user = User(name="test", image="iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==")
db.session.add(user)
db.session.commit()
return app
host = "0.0.0.0"
app = create_app(host=host)
#app.route("/image/<string:user_name>")
def show_profile(user_name):
user = User.query.filter(User.name==user_name).one_or_none()
if user:
return f'User: {user.name}<br/>Pic:<img src="data:image/png;base64, {user.image}"/>'
return "Not Found", 404
if __name__ == "__main__":
app.run(host=host)
then go to
http://localhost:5000/image/test
AWS Sagemaker model deployment is failing when endpoint_name argument is specified. Any thoughts?
Without endpoint_name argument in deploy, model deployment works successfully.
Model training and saving into S3 location is successful either way.
import boto3
import os
import sagemaker
from sagemaker import get_execution_role
from sagemaker.predictor import csv_serializer
from sagemaker.amazon.amazon_estimator import get_image_uri
bucket = 'Y'
prefix = 'Z'
role = get_execution_role()
train_data, validation_data, test_data = np.split(df.sample(frac=1, random_state=100), [int(0.5 * len(df)), int(0.8 * len(df))])
train_data.to_csv('train.csv', index=False, header=False)
validation_data.to_csv('validation.csv', index=False, header=False)
test_data.to_csv('test.csv', index=False)
boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'train/X/train.csv')).upload_file('train.csv')
boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'validation/X/validation.csv')).upload_file('validation.csv')
container = get_image_uri(boto3.Session().region_name, 'xgboost')
#print(container)
s3_input_train = sagemaker.s3_input(s3_data='s3://{}/{}/train/{}'.format(bucket, prefix, suffix), content_type='csv')
s3_input_validation = sagemaker.s3_input(s3_data='s3://{}/{}/validation/{}/'.format(bucket, prefix, suffix), content_type='csv')
sess = sagemaker.Session()
output_loc = 's3://{}/{}/output'.format(bucket, prefix)
xgb = sagemaker.estimator.Estimator(container,
role,
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
output_path=output_loc,
sagemaker_session=sess,
base_job_name='X')
#print('Model output to: {}'.format(output_location))
xgb.set_hyperparameters(eta=0.5,
objective='reg:linear',
eval_metric='rmse',
max_depth=3,
min_child_weight=1,
gamma=0,
early_stopping_rounds=10,
subsample=0.8,
colsample_bytree=0.8,
num_round=1000)
#Model fitting
xgb.fit({'train': s3_input_train, 'validation': s3_input_validation})
#Deploy model with automatic endpoint created
xgb_predictor_X = xgb.deploy(initial_instance_count=1, instance_type='ml.m4.xlarge', endpoint_name='X')
xgb_predictor_X.content_type = 'text/csv'
xgb_predictor_X.serializer = csv_serializer
xgb_predictor_X.deserializer = None
INFO:sagemaker:Creating endpoint with name delaymins
ClientError: An error occurred (ValidationException) when calling the CreateEndpoint operation: Could not find model "arn:aws:sagemaker:us-west-2::model/X-2019-01-08-18-17-42-158".
Figured it out! If custom endpoint name is not ended before redeploying it, it get blacklisted(not sure if this is temporary). Therefore a different endpoint name must be used if this mistake is made. Moral of the story: Always end an endpoint before redeploying.
I want to get unread mails from yesterday. So i had to implement multiple queries in the message.list function which give me an error of invalid syntax. How do i do it? Can someone help me? And will internalDate help me anyway?
from __future__ import print_function
import httplib2
import os
from email.utils import parsedate_tz,mktime_tz,formatdate
from requests.adapters import HTTPAdapter
import datetime
from datetime import date,timedelta
import time
from apiclient import discovery
import oauth2client
from oauth2client import client
from oauth2client import tools
import json
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = 'client_server.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'
def get_credentials():
"""Gets valid user credentials from storage.
If nothing has been stored, or if the stored credentials are invalid,
the OAuth2 flow is completed to obtain the new credentials.
Returns:
Credentials, the obtained credential.
"""
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'gmail-python-quickstart.json')
store = oauth2client.file.Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
if flags:
credentials = tools.run_flow(flow, store, flags)
else: # Needed only for compatibility with Python 2.6
credentials = tools.run(flow, store)
print('Storing credentials to ' + credential_path)
return credentials
def main():
da=date.fromordinal(730920)
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('gmail', 'v1', http=http)
today=date.today()
print (today)
yesterday=today-timedelta(1)
print (yesterday)
response = service.users().messages().list(userId='me',q='{in:inbox is:unread} AND {after: {0}.format(yesterday.strftime('%Y/%m/%d'))}').execute()
messages=[]
store=[]
message1=[]
test2=[]
da=[]
if 'messages' in response:
messages.extend(response['messages'])
fo = open("foo.txt", "wb")
for i in range(len(messages)):
store=messages[i]['id']
message = service.users().messages().get(userId='me',id=store,format='metadata',metadataHeaders=['from','date']).execute()
fo.write(store+" ");
#print(message['payload']['headers'][0])
fo.write(message['snippet'].encode('utf-8')+" ")
if message['payload']['headers'][0]['name'].lower()=="from":
From=message['payload']['headers'][0]['value']
fo.write(From+" ");
elif message['payload']['headers'][0]['name'].lower()=="date":
da=message['payload']['headers'][0]['value']
fo.write(da+"\n");
for line in open("foo.txt"):
print(line)
# Open a file
# Close opend file
fo.close()
if __name__ == '__main__':
main()
Use:
q='in:inbox is:unread newer_than:3d'
as the query. Gmail queries don't have a concept of timezones so if you try to get one day worth of email you'll end up with some overlap. Just use local filtering to narrow those down. See advanced Gmail search for more help. The API and Gmail UI use the same query syntax and should show the same results so you can do testing in the UI.
how can i modify the following python code to upload the ecg.txt to google drive every 5 seconds where i update this text file. the code now ask for Oauth every time it upload the file , i want it to ask for authentication only in the first time.
#!/usr/bin/python
import httplib2
import pprint
from apiclient.discovery import build
from apiclient.http import MediaFileUpload
from oauth2client.client import OAuth2WebServerFlow
# Copy your credentials from the console
CLIENT_ID = XXXXXX
CLIENT_SECRET = XXXXX
# Check https://developers.google.com/drive/scopes for all available scopes
OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive'
# Redirect URI for installed apps
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
# Path to the file to upload
FILENAME = 'ecg.txt'
# Run through the OAuth flow and retrieve credentials
flow = OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, OAUTH_SCOPE, REDIRECT_URI)
authorize_url = flow.step1_get_authorize_url()
print 'Go to the following link in your browser: ' + authorize_url
code = raw_input('Enter verification code: ').strip()
credentials = flow.step2_exchange(code)
# Create an httplib2.Http object and authorize it with our credentials
http = httplib2.Http()
http = credentials.authorize(http)
drive_service = build('drive', 'v2', http=http)
# Insert a file
media_body = MediaFileUpload(FILENAME, mimetype='text/plain', resumable=True)
body = {
'title': 'My document',
'description': 'A test document',
'mimeType': 'text/plain'
}
file = drive_service.files().insert(body=body, media_body=media_body).execute()
pprint.pprint(file)
If you use the decorator as seen in the video bellow it stores your access token in the db for you and handles refreshing when needed.
https://www.youtube.com/watch?v=HoUdWBzUZ-M
https://developers.google.com/api-client-library/python/guide/google_app_engine
from oauth2client.appengine import OAuth2Decorator
decorator = OAuth2Decorator(client_id=CLIENT_ID,
client_secret=CLIENT_SECRET,
scope=OAUTH_SCOPE,
callback_path=REDIRECT_URI)
class MainPage(webapp2.RequestHandler):
#decorator.oauth_required #Simply place this above any function that requires login.
def get(self):
I'd been searching for a way to do cookie based authentication/sessions in Google App Engine because I don't like the idea of memcache based sessions, and I also don't like the idea of forcing users to create google accounts just to use a website. I stumbled across someone's posting that mentioned some signed cookie functions from the Tornado framework and it looks like what I need. What I have in mind is storing a user's id in a tamper proof cookie, and maybe using a decorator for the request handlers to test the authentication status of the user, and as a side benefit the user id will be available to the request handler for datastore work and such. The concept would be similar to forms authentication in ASP.NET. This code comes from the web.py module of the Tornado framework.
According to the docstrings, it "Signs and timestamps a cookie so it cannot be forged" and
"Returns the given signed cookie if it validates, or None."
I've tried to use it in an App Engine Project, but I don't understand the nuances of trying to get these methods to work in the context of the request handler. Can someone show me the right way to do this without losing the functionality that the FriendFeed developers put into it? The set_secure_cookie, and get_secure_cookie portions are the most important, but it would be nice to be able to use the other methods as well.
#!/usr/bin/env python
import Cookie
import base64
import time
import hashlib
import hmac
import datetime
import re
import calendar
import email.utils
import logging
def _utf8(s):
if isinstance(s, unicode):
return s.encode("utf-8")
assert isinstance(s, str)
return s
def _unicode(s):
if isinstance(s, str):
try:
return s.decode("utf-8")
except UnicodeDecodeError:
raise HTTPError(400, "Non-utf8 argument")
assert isinstance(s, unicode)
return s
def _time_independent_equals(a, b):
if len(a) != len(b):
return False
result = 0
for x, y in zip(a, b):
result |= ord(x) ^ ord(y)
return result == 0
def cookies(self):
"""A dictionary of Cookie.Morsel objects."""
if not hasattr(self,"_cookies"):
self._cookies = Cookie.BaseCookie()
if "Cookie" in self.request.headers:
try:
self._cookies.load(self.request.headers["Cookie"])
except:
self.clear_all_cookies()
return self._cookies
def _cookie_signature(self,*parts):
self.require_setting("cookie_secret","secure cookies")
hash = hmac.new(self.application.settings["cookie_secret"],
digestmod=hashlib.sha1)
for part in parts:hash.update(part)
return hash.hexdigest()
def get_cookie(self,name,default=None):
"""Gets the value of the cookie with the given name,else default."""
if name in self.cookies:
return self.cookies[name].value
return default
def set_cookie(self,name,value,domain=None,expires=None,path="/",
expires_days=None):
"""Sets the given cookie name/value with the given options."""
name = _utf8(name)
value = _utf8(value)
if re.search(r"[\x00-\x20]",name + value):
# Don't let us accidentally inject bad stuff
raise ValueError("Invalid cookie %r:%r" % (name,value))
if not hasattr(self,"_new_cookies"):
self._new_cookies = []
new_cookie = Cookie.BaseCookie()
self._new_cookies.append(new_cookie)
new_cookie[name] = value
if domain:
new_cookie[name]["domain"] = domain
if expires_days is not None and not expires:
expires = datetime.datetime.utcnow() + datetime.timedelta(
days=expires_days)
if expires:
timestamp = calendar.timegm(expires.utctimetuple())
new_cookie[name]["expires"] = email.utils.formatdate(
timestamp,localtime=False,usegmt=True)
if path:
new_cookie[name]["path"] = path
def clear_cookie(self,name,path="/",domain=None):
"""Deletes the cookie with the given name."""
expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
self.set_cookie(name,value="",path=path,expires=expires,
domain=domain)
def clear_all_cookies(self):
"""Deletes all the cookies the user sent with this request."""
for name in self.cookies.iterkeys():
self.clear_cookie(name)
def set_secure_cookie(self,name,value,expires_days=30,**kwargs):
"""Signs and timestamps a cookie so it cannot be forged"""
timestamp = str(int(time.time()))
value = base64.b64encode(value)
signature = self._cookie_signature(name,value,timestamp)
value = "|".join([value,timestamp,signature])
self.set_cookie(name,value,expires_days=expires_days,**kwargs)
def get_secure_cookie(self,name,include_name=True,value=None):
"""Returns the given signed cookie if it validates,or None"""
if value is None:value = self.get_cookie(name)
if not value:return None
parts = value.split("|")
if len(parts) != 3:return None
if include_name:
signature = self._cookie_signature(name,parts[0],parts[1])
else:
signature = self._cookie_signature(parts[0],parts[1])
if not _time_independent_equals(parts[2],signature):
logging.warning("Invalid cookie signature %r",value)
return None
timestamp = int(parts[1])
if timestamp < time.time() - 31 * 86400:
logging.warning("Expired cookie %r",value)
return None
try:
return base64.b64decode(parts[0])
except:
return None
uid=1234|1234567890|d32b9e9c67274fa062e2599fd659cc14
Parts:
1. uid is the name of the key
2. 1234 is your value in clear
3. 1234567890 is the timestamp
4. d32b9e9c67274fa062e2599fd659cc14 is the signature made from the value and the timestamp
Tornado was never meant to work with App Engine (it's "its own server" through and through). Why don't you pick instead some framework that was meant for App Engine from the word "go" and is lightweight and dandy, such as tipfy? It gives you authentication using its own user system or any of App Engine's own users, OpenIn, OAuth, and Facebook; sessions with secure cookies or GAE datastore; and much more besides, all in a superbly lightweight "non-framework" approach based on WSGI and Werkzeug. What's not to like?!
For those who are still looking, we've extracted just the Tornado cookie implementation that you can use with App Engine at ThriveSmart. We're using it successfully on App Engine and will continue to keep it updated.
The cookie library itself is at:
http://github.com/thrivesmart/prayls/blob/master/prayls/lilcookies.py
You can see it in action in our example app that's included. If the structure of our repository ever changes, you can look for lilcookes.py within github.com/thrivesmart/prayls
I hope that's helpful to someone out there!
This works if anyone is interested:
from google.appengine.ext import webapp
import Cookie
import base64
import time
import hashlib
import hmac
import datetime
import re
import calendar
import email.utils
import logging
def _utf8(s):
if isinstance(s, unicode):
return s.encode("utf-8")
assert isinstance(s, str)
return s
def _unicode(s):
if isinstance(s, str):
try:
return s.decode("utf-8")
except UnicodeDecodeError:
raise HTTPError(400, "Non-utf8 argument")
assert isinstance(s, unicode)
return s
def _time_independent_equals(a, b):
if len(a) != len(b):
return False
result = 0
for x, y in zip(a, b):
result |= ord(x) ^ ord(y)
return result == 0
class ExtendedRequestHandler(webapp.RequestHandler):
"""Extends the Google App Engine webapp.RequestHandler."""
def clear_cookie(self,name,path="/",domain=None):
"""Deletes the cookie with the given name."""
expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
self.set_cookie(name,value="",path=path,expires=expires,
domain=domain)
def clear_all_cookies(self):
"""Deletes all the cookies the user sent with this request."""
for name in self.cookies.iterkeys():
self.clear_cookie(name)
def cookies(self):
"""A dictionary of Cookie.Morsel objects."""
if not hasattr(self,"_cookies"):
self._cookies = Cookie.BaseCookie()
if "Cookie" in self.request.headers:
try:
self._cookies.load(self.request.headers["Cookie"])
except:
self.clear_all_cookies()
return self._cookies
def _cookie_signature(self,*parts):
"""Hashes a string based on a pass-phrase."""
hash = hmac.new("MySecretPhrase",digestmod=hashlib.sha1)
for part in parts:hash.update(part)
return hash.hexdigest()
def get_cookie(self,name,default=None):
"""Gets the value of the cookie with the given name,else default."""
if name in self.request.cookies:
return self.request.cookies[name]
return default
def set_cookie(self,name,value,domain=None,expires=None,path="/",expires_days=None):
"""Sets the given cookie name/value with the given options."""
name = _utf8(name)
value = _utf8(value)
if re.search(r"[\x00-\x20]",name + value): # Don't let us accidentally inject bad stuff
raise ValueError("Invalid cookie %r:%r" % (name,value))
new_cookie = Cookie.BaseCookie()
new_cookie[name] = value
if domain:
new_cookie[name]["domain"] = domain
if expires_days is not None and not expires:
expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days)
if expires:
timestamp = calendar.timegm(expires.utctimetuple())
new_cookie[name]["expires"] = email.utils.formatdate(timestamp,localtime=False,usegmt=True)
if path:
new_cookie[name]["path"] = path
for morsel in new_cookie.values():
self.response.headers.add_header('Set-Cookie',morsel.OutputString(None))
def set_secure_cookie(self,name,value,expires_days=30,**kwargs):
"""Signs and timestamps a cookie so it cannot be forged"""
timestamp = str(int(time.time()))
value = base64.b64encode(value)
signature = self._cookie_signature(name,value,timestamp)
value = "|".join([value,timestamp,signature])
self.set_cookie(name,value,expires_days=expires_days,**kwargs)
def get_secure_cookie(self,name,include_name=True,value=None):
"""Returns the given signed cookie if it validates,or None"""
if value is None:value = self.get_cookie(name)
if not value:return None
parts = value.split("|")
if len(parts) != 3:return None
if include_name:
signature = self._cookie_signature(name,parts[0],parts[1])
else:
signature = self._cookie_signature(parts[0],parts[1])
if not _time_independent_equals(parts[2],signature):
logging.warning("Invalid cookie signature %r",value)
return None
timestamp = int(parts[1])
if timestamp < time.time() - 31 * 86400:
logging.warning("Expired cookie %r",value)
return None
try:
return base64.b64decode(parts[0])
except:
return None
It can be used like this:
class MyHandler(ExtendedRequestHandler):
def get(self):
self.set_cookie(name="MyCookie",value="NewValue",expires_days=10)
self.set_secure_cookie(name="MySecureCookie",value="SecureValue",expires_days=10)
value1 = self.get_cookie('MyCookie')
value2 = self.get_secure_cookie('MySecureCookie')
If you only want to store the user's user ID in the cookie (presumably so you can look their record up in the datastore), you don't need 'secure' or tamper-proof cookies - you just need a namespace that's big enough to make guessing user IDs impractical - eg, GUIDs, or other random data.
One pre-made option for this, which uses the datastore for session storage, is Beaker. Alternately, you could handle this yourself with set-cookie/cookie headers, if you really just need to store their user ID.
Someone recently extracted the authentication and session code from Tornado and created a new library specifically for GAE.
Perhaps this is more then you need, but since they did it specifically for GAE you shouldn't have to worry about adapting it yourself.
Their library is called gaema. Here is their announcement in the GAE Python group on 4 Mar 2010:
http://groups.google.com/group/google-appengine-python/browse_thread/thread/d2d6c597d66ecad3/06c6dc49cb8eca0c?lnk=gst&q=tornado#06c6dc49cb8eca0c