I'm using gmail API to insert an email to the inbox. When the From domain is outside the organization, I expect the external label to be added but it's not.
I'm using python and everything else is working, I am using a service account and able to impersonate on behalf the user's email and I'm using labelIds of ['INBOX', 'UNREAD'] and I need the newly external label as well but couldn't figure a way to add it through the API.
This feature is turned ON for the workspace.
Update - adding my code:
from googleapiclient import discovery, errors
from google.oauth2 import service_account
from email.mime.text import MIMEText
import base64
SERVICE_ACCOUNT_FILE = 'insert-messages-91e77b62878f.json'
SCOPES = ['https://www.googleapis.com/auth/gmail.insert']
def validationService():
# Set the credentials
credentials = service_account.Credentials.\
from_service_account_file(SERVICE_ACCOUNT_FILE, scopes= SCOPES)
# Delegate the credentials to the user you want to impersonate
delegated_credentials = credentials.with_subject('<some_user>')
service = discovery.build('gmail', 'v1', credentials=delegated_credentials)
return service
def SendMessage(service, message):
message = service.users().messages().insert(userId='me', body=message).execute() # me will use <some_user> from above
return message
def CreateMessage(sender, to, subject, message_text):
message = MIMEText(message_text)
message['To'] = to
message['From'] = sender
message['Subject'] = subject
return {'raw': base64.urlsafe_b64encode(message.as_string().encode()).decode(), 'labelIds': ['INBOX', 'UNREAD']}
def main():
try:
service = validationService()
email = CreateMessage('some#external.com', "<some_user>", "Test", "This is a test")
email_sent = SendMessage(service, email)
print('Message Id:', email_sent['id'])
except errors.HttpError as err:
print('\n---------------You have the following error-------------')
print(err)
print('---------------You have the following error-------------\n')
if __name__ == '__main__':
main()
AWS Sagemaker model deployment is failing when endpoint_name argument is specified. Any thoughts?
Without endpoint_name argument in deploy, model deployment works successfully.
Model training and saving into S3 location is successful either way.
import boto3
import os
import sagemaker
from sagemaker import get_execution_role
from sagemaker.predictor import csv_serializer
from sagemaker.amazon.amazon_estimator import get_image_uri
bucket = 'Y'
prefix = 'Z'
role = get_execution_role()
train_data, validation_data, test_data = np.split(df.sample(frac=1, random_state=100), [int(0.5 * len(df)), int(0.8 * len(df))])
train_data.to_csv('train.csv', index=False, header=False)
validation_data.to_csv('validation.csv', index=False, header=False)
test_data.to_csv('test.csv', index=False)
boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'train/X/train.csv')).upload_file('train.csv')
boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'validation/X/validation.csv')).upload_file('validation.csv')
container = get_image_uri(boto3.Session().region_name, 'xgboost')
#print(container)
s3_input_train = sagemaker.s3_input(s3_data='s3://{}/{}/train/{}'.format(bucket, prefix, suffix), content_type='csv')
s3_input_validation = sagemaker.s3_input(s3_data='s3://{}/{}/validation/{}/'.format(bucket, prefix, suffix), content_type='csv')
sess = sagemaker.Session()
output_loc = 's3://{}/{}/output'.format(bucket, prefix)
xgb = sagemaker.estimator.Estimator(container,
role,
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
output_path=output_loc,
sagemaker_session=sess,
base_job_name='X')
#print('Model output to: {}'.format(output_location))
xgb.set_hyperparameters(eta=0.5,
objective='reg:linear',
eval_metric='rmse',
max_depth=3,
min_child_weight=1,
gamma=0,
early_stopping_rounds=10,
subsample=0.8,
colsample_bytree=0.8,
num_round=1000)
#Model fitting
xgb.fit({'train': s3_input_train, 'validation': s3_input_validation})
#Deploy model with automatic endpoint created
xgb_predictor_X = xgb.deploy(initial_instance_count=1, instance_type='ml.m4.xlarge', endpoint_name='X')
xgb_predictor_X.content_type = 'text/csv'
xgb_predictor_X.serializer = csv_serializer
xgb_predictor_X.deserializer = None
INFO:sagemaker:Creating endpoint with name delaymins
ClientError: An error occurred (ValidationException) when calling the CreateEndpoint operation: Could not find model "arn:aws:sagemaker:us-west-2::model/X-2019-01-08-18-17-42-158".
Figured it out! If custom endpoint name is not ended before redeploying it, it get blacklisted(not sure if this is temporary). Therefore a different endpoint name must be used if this mistake is made. Moral of the story: Always end an endpoint before redeploying.
how can i modify the following python code to upload the ecg.txt to google drive every 5 seconds where i update this text file. the code now ask for Oauth every time it upload the file , i want it to ask for authentication only in the first time.
#!/usr/bin/python
import httplib2
import pprint
from apiclient.discovery import build
from apiclient.http import MediaFileUpload
from oauth2client.client import OAuth2WebServerFlow
# Copy your credentials from the console
CLIENT_ID = XXXXXX
CLIENT_SECRET = XXXXX
# Check https://developers.google.com/drive/scopes for all available scopes
OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive'
# Redirect URI for installed apps
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
# Path to the file to upload
FILENAME = 'ecg.txt'
# Run through the OAuth flow and retrieve credentials
flow = OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, OAUTH_SCOPE, REDIRECT_URI)
authorize_url = flow.step1_get_authorize_url()
print 'Go to the following link in your browser: ' + authorize_url
code = raw_input('Enter verification code: ').strip()
credentials = flow.step2_exchange(code)
# Create an httplib2.Http object and authorize it with our credentials
http = httplib2.Http()
http = credentials.authorize(http)
drive_service = build('drive', 'v2', http=http)
# Insert a file
media_body = MediaFileUpload(FILENAME, mimetype='text/plain', resumable=True)
body = {
'title': 'My document',
'description': 'A test document',
'mimeType': 'text/plain'
}
file = drive_service.files().insert(body=body, media_body=media_body).execute()
pprint.pprint(file)
If you use the decorator as seen in the video bellow it stores your access token in the db for you and handles refreshing when needed.
https://www.youtube.com/watch?v=HoUdWBzUZ-M
https://developers.google.com/api-client-library/python/guide/google_app_engine
from oauth2client.appengine import OAuth2Decorator
decorator = OAuth2Decorator(client_id=CLIENT_ID,
client_secret=CLIENT_SECRET,
scope=OAUTH_SCOPE,
callback_path=REDIRECT_URI)
class MainPage(webapp2.RequestHandler):
#decorator.oauth_required #Simply place this above any function that requires login.
def get(self):
I am trying to scrape some web content to get stastics from google appstats that my google app generated. Please note this is different then google analytics. I am using python 2.7.5.
The issue I am facing is the initial google authentication in my request.
I have the apis I need to call from google app stats but I keep getting a DENY in response back when I am using my own google appengine account credentials. This results in a redirect to accounts.google.com page.
I have tried several different approaches without a successfull login to accounts.google.com.
Anyone have any ideas on this?
More helpfull would be if you can point me to some good reference material
Thanks
This code sample will allow you to get content of /secure page which protected by google login. Do not forget to set email, password and application ID. Then you can fetch other protected pages using this opener.
import urllib
import urllib2
import cookielib
import logging
EMAIL = ''
PASSWORD = ''
APPID = 'YOURAPPID'
# Setup to be able to get the needed cookies that GAE returns
cookiejar = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
urllib2.install_opener(opener)
# This is the setup to construct the login URL for authentication.
authreq_data = urllib.urlencode({'Email': EMAIL,
'Passwd': PASSWORD,
'service': 'ah',
'source': '',
'accountType': 'HOSTED_OR_GOOGLE'})
# Get an AuthToken from Google Accounts
auth_req = urllib2.Request('https://www.google.com/accounts/ClientLogin',
data=authreq_data)
try:
auth_resp = opener.open(auth_req)
logging.info('Successful authorization as %s' % EMAIL)
except urllib2.HTTPError:
logging.warning('Authorization as %s failed. '
'Please, check your email and password' % EMAIL)
auth_resp_body = auth_resp.read()
auth_resp_dict = dict(x.split('=')
for x in auth_resp_body.split('\n') if x)
authtoken = auth_resp_dict['Auth']
authreq_data = urllib.urlencode({'continue': 'http://%s.appspot.com/secure' % APPID,
'auth': authtoken})
login_uri = ('http://%s.appspot.com/_ah/login?%s' % (APPID, authreq_data))
# Do the actual login and getting the cookies.
print opener.open(urllib2.Request(login_uri)).read()
I'd been searching for a way to do cookie based authentication/sessions in Google App Engine because I don't like the idea of memcache based sessions, and I also don't like the idea of forcing users to create google accounts just to use a website. I stumbled across someone's posting that mentioned some signed cookie functions from the Tornado framework and it looks like what I need. What I have in mind is storing a user's id in a tamper proof cookie, and maybe using a decorator for the request handlers to test the authentication status of the user, and as a side benefit the user id will be available to the request handler for datastore work and such. The concept would be similar to forms authentication in ASP.NET. This code comes from the web.py module of the Tornado framework.
According to the docstrings, it "Signs and timestamps a cookie so it cannot be forged" and
"Returns the given signed cookie if it validates, or None."
I've tried to use it in an App Engine Project, but I don't understand the nuances of trying to get these methods to work in the context of the request handler. Can someone show me the right way to do this without losing the functionality that the FriendFeed developers put into it? The set_secure_cookie, and get_secure_cookie portions are the most important, but it would be nice to be able to use the other methods as well.
#!/usr/bin/env python
import Cookie
import base64
import time
import hashlib
import hmac
import datetime
import re
import calendar
import email.utils
import logging
def _utf8(s):
if isinstance(s, unicode):
return s.encode("utf-8")
assert isinstance(s, str)
return s
def _unicode(s):
if isinstance(s, str):
try:
return s.decode("utf-8")
except UnicodeDecodeError:
raise HTTPError(400, "Non-utf8 argument")
assert isinstance(s, unicode)
return s
def _time_independent_equals(a, b):
if len(a) != len(b):
return False
result = 0
for x, y in zip(a, b):
result |= ord(x) ^ ord(y)
return result == 0
def cookies(self):
"""A dictionary of Cookie.Morsel objects."""
if not hasattr(self,"_cookies"):
self._cookies = Cookie.BaseCookie()
if "Cookie" in self.request.headers:
try:
self._cookies.load(self.request.headers["Cookie"])
except:
self.clear_all_cookies()
return self._cookies
def _cookie_signature(self,*parts):
self.require_setting("cookie_secret","secure cookies")
hash = hmac.new(self.application.settings["cookie_secret"],
digestmod=hashlib.sha1)
for part in parts:hash.update(part)
return hash.hexdigest()
def get_cookie(self,name,default=None):
"""Gets the value of the cookie with the given name,else default."""
if name in self.cookies:
return self.cookies[name].value
return default
def set_cookie(self,name,value,domain=None,expires=None,path="/",
expires_days=None):
"""Sets the given cookie name/value with the given options."""
name = _utf8(name)
value = _utf8(value)
if re.search(r"[\x00-\x20]",name + value):
# Don't let us accidentally inject bad stuff
raise ValueError("Invalid cookie %r:%r" % (name,value))
if not hasattr(self,"_new_cookies"):
self._new_cookies = []
new_cookie = Cookie.BaseCookie()
self._new_cookies.append(new_cookie)
new_cookie[name] = value
if domain:
new_cookie[name]["domain"] = domain
if expires_days is not None and not expires:
expires = datetime.datetime.utcnow() + datetime.timedelta(
days=expires_days)
if expires:
timestamp = calendar.timegm(expires.utctimetuple())
new_cookie[name]["expires"] = email.utils.formatdate(
timestamp,localtime=False,usegmt=True)
if path:
new_cookie[name]["path"] = path
def clear_cookie(self,name,path="/",domain=None):
"""Deletes the cookie with the given name."""
expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
self.set_cookie(name,value="",path=path,expires=expires,
domain=domain)
def clear_all_cookies(self):
"""Deletes all the cookies the user sent with this request."""
for name in self.cookies.iterkeys():
self.clear_cookie(name)
def set_secure_cookie(self,name,value,expires_days=30,**kwargs):
"""Signs and timestamps a cookie so it cannot be forged"""
timestamp = str(int(time.time()))
value = base64.b64encode(value)
signature = self._cookie_signature(name,value,timestamp)
value = "|".join([value,timestamp,signature])
self.set_cookie(name,value,expires_days=expires_days,**kwargs)
def get_secure_cookie(self,name,include_name=True,value=None):
"""Returns the given signed cookie if it validates,or None"""
if value is None:value = self.get_cookie(name)
if not value:return None
parts = value.split("|")
if len(parts) != 3:return None
if include_name:
signature = self._cookie_signature(name,parts[0],parts[1])
else:
signature = self._cookie_signature(parts[0],parts[1])
if not _time_independent_equals(parts[2],signature):
logging.warning("Invalid cookie signature %r",value)
return None
timestamp = int(parts[1])
if timestamp < time.time() - 31 * 86400:
logging.warning("Expired cookie %r",value)
return None
try:
return base64.b64decode(parts[0])
except:
return None
uid=1234|1234567890|d32b9e9c67274fa062e2599fd659cc14
Parts:
1. uid is the name of the key
2. 1234 is your value in clear
3. 1234567890 is the timestamp
4. d32b9e9c67274fa062e2599fd659cc14 is the signature made from the value and the timestamp
Tornado was never meant to work with App Engine (it's "its own server" through and through). Why don't you pick instead some framework that was meant for App Engine from the word "go" and is lightweight and dandy, such as tipfy? It gives you authentication using its own user system or any of App Engine's own users, OpenIn, OAuth, and Facebook; sessions with secure cookies or GAE datastore; and much more besides, all in a superbly lightweight "non-framework" approach based on WSGI and Werkzeug. What's not to like?!
For those who are still looking, we've extracted just the Tornado cookie implementation that you can use with App Engine at ThriveSmart. We're using it successfully on App Engine and will continue to keep it updated.
The cookie library itself is at:
http://github.com/thrivesmart/prayls/blob/master/prayls/lilcookies.py
You can see it in action in our example app that's included. If the structure of our repository ever changes, you can look for lilcookes.py within github.com/thrivesmart/prayls
I hope that's helpful to someone out there!
This works if anyone is interested:
from google.appengine.ext import webapp
import Cookie
import base64
import time
import hashlib
import hmac
import datetime
import re
import calendar
import email.utils
import logging
def _utf8(s):
if isinstance(s, unicode):
return s.encode("utf-8")
assert isinstance(s, str)
return s
def _unicode(s):
if isinstance(s, str):
try:
return s.decode("utf-8")
except UnicodeDecodeError:
raise HTTPError(400, "Non-utf8 argument")
assert isinstance(s, unicode)
return s
def _time_independent_equals(a, b):
if len(a) != len(b):
return False
result = 0
for x, y in zip(a, b):
result |= ord(x) ^ ord(y)
return result == 0
class ExtendedRequestHandler(webapp.RequestHandler):
"""Extends the Google App Engine webapp.RequestHandler."""
def clear_cookie(self,name,path="/",domain=None):
"""Deletes the cookie with the given name."""
expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
self.set_cookie(name,value="",path=path,expires=expires,
domain=domain)
def clear_all_cookies(self):
"""Deletes all the cookies the user sent with this request."""
for name in self.cookies.iterkeys():
self.clear_cookie(name)
def cookies(self):
"""A dictionary of Cookie.Morsel objects."""
if not hasattr(self,"_cookies"):
self._cookies = Cookie.BaseCookie()
if "Cookie" in self.request.headers:
try:
self._cookies.load(self.request.headers["Cookie"])
except:
self.clear_all_cookies()
return self._cookies
def _cookie_signature(self,*parts):
"""Hashes a string based on a pass-phrase."""
hash = hmac.new("MySecretPhrase",digestmod=hashlib.sha1)
for part in parts:hash.update(part)
return hash.hexdigest()
def get_cookie(self,name,default=None):
"""Gets the value of the cookie with the given name,else default."""
if name in self.request.cookies:
return self.request.cookies[name]
return default
def set_cookie(self,name,value,domain=None,expires=None,path="/",expires_days=None):
"""Sets the given cookie name/value with the given options."""
name = _utf8(name)
value = _utf8(value)
if re.search(r"[\x00-\x20]",name + value): # Don't let us accidentally inject bad stuff
raise ValueError("Invalid cookie %r:%r" % (name,value))
new_cookie = Cookie.BaseCookie()
new_cookie[name] = value
if domain:
new_cookie[name]["domain"] = domain
if expires_days is not None and not expires:
expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days)
if expires:
timestamp = calendar.timegm(expires.utctimetuple())
new_cookie[name]["expires"] = email.utils.formatdate(timestamp,localtime=False,usegmt=True)
if path:
new_cookie[name]["path"] = path
for morsel in new_cookie.values():
self.response.headers.add_header('Set-Cookie',morsel.OutputString(None))
def set_secure_cookie(self,name,value,expires_days=30,**kwargs):
"""Signs and timestamps a cookie so it cannot be forged"""
timestamp = str(int(time.time()))
value = base64.b64encode(value)
signature = self._cookie_signature(name,value,timestamp)
value = "|".join([value,timestamp,signature])
self.set_cookie(name,value,expires_days=expires_days,**kwargs)
def get_secure_cookie(self,name,include_name=True,value=None):
"""Returns the given signed cookie if it validates,or None"""
if value is None:value = self.get_cookie(name)
if not value:return None
parts = value.split("|")
if len(parts) != 3:return None
if include_name:
signature = self._cookie_signature(name,parts[0],parts[1])
else:
signature = self._cookie_signature(parts[0],parts[1])
if not _time_independent_equals(parts[2],signature):
logging.warning("Invalid cookie signature %r",value)
return None
timestamp = int(parts[1])
if timestamp < time.time() - 31 * 86400:
logging.warning("Expired cookie %r",value)
return None
try:
return base64.b64decode(parts[0])
except:
return None
It can be used like this:
class MyHandler(ExtendedRequestHandler):
def get(self):
self.set_cookie(name="MyCookie",value="NewValue",expires_days=10)
self.set_secure_cookie(name="MySecureCookie",value="SecureValue",expires_days=10)
value1 = self.get_cookie('MyCookie')
value2 = self.get_secure_cookie('MySecureCookie')
If you only want to store the user's user ID in the cookie (presumably so you can look their record up in the datastore), you don't need 'secure' or tamper-proof cookies - you just need a namespace that's big enough to make guessing user IDs impractical - eg, GUIDs, or other random data.
One pre-made option for this, which uses the datastore for session storage, is Beaker. Alternately, you could handle this yourself with set-cookie/cookie headers, if you really just need to store their user ID.
Someone recently extracted the authentication and session code from Tornado and created a new library specifically for GAE.
Perhaps this is more then you need, but since they did it specifically for GAE you shouldn't have to worry about adapting it yourself.
Their library is called gaema. Here is their announcement in the GAE Python group on 4 Mar 2010:
http://groups.google.com/group/google-appengine-python/browse_thread/thread/d2d6c597d66ecad3/06c6dc49cb8eca0c?lnk=gst&q=tornado#06c6dc49cb8eca0c