KeyError during python GAE push task queue with cron - google-app-engine

Question

I'm very close to finishing up a project that uses push task queues in GAE to send out follow-up emails to users. However, I keep getting a KeyError and don't know why. I been looking for good models to base my project on but haven't found any decent examples that use multiple parameters. The GAE documentation has improved in the last month but still leaves a lot to be desired.
I've checked many pieces of the code using the interactive console in the development server but still I don't know what I am doing wrong. My best guess is that the parameters are not getting passed along to the next part of the script (class pushQueue).
app.yaml:
application: gae-project
version: 1
runtime: python27
api_version: 1
threadsafe: true
handlers:
- url: /cron/sendfu
script: main.app
login: admin
- url: /emailworker
script: main.app
login: admin
- url: /worker
script: main.app
login: admin
- url: /.*
script: main.app
login: required
cron.yaml:
cron:
- description: sends follow-up emails
url: /cron/sendfu
schedule: every day 20:00
queue.yaml:
total_storage_limit: 120M
queue:
- name: emailworker
rate: 1/s
bucket_size: 50
retry_parameters:
task_retry_limit: 5
task_age_limit: 6h
min_backoff_seconds: 10
max_backoff_seconds: 60
main.py:
import webapp2
import datetime
from google.appengine.ext import db
from google.appengine.api import users
from google.appengine.api import taskqueue
import jinja2
import os
jinja_environment = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))
class emailJobs(db.Model):
""" Models an a list of email jobs for each user """
triggerid = db.StringProperty() #Trig id
recipientid_po = db.StringProperty() # id
recipientlang = db.StringProperty() #Language
fu_email_sent = db.DateTimeProperty()
fuperiod = db.IntegerProperty() # (0 - 13)
fu1 = db.DateTimeProperty()
fu2 = db.DateTimeProperty()
#classmethod
def update_fusent(cls, key_name, senddate):
""" Class method that updates fu messages sent in the GAE Datastore """
emailsjobs = cls.get_by_key_name(key_name)
if emailsjobs is None:
emailsjobs = cls(key_name=key_name)
emailsjobs.fu_email_sent = senddate
emailsjobs.put()
def timeStampFM(now):
d = now.date()
year = d.year
month = d.month
day = d.day
t = now.time()
hour = t.hour
minute = t.minute + 5
second = t.second
today_datetime = datetime.datetime(year, month, day, hour, minute, second)
return today_datetime
class MainPage(webapp2.RequestHandler):
""" Main admin login page """
def get(self):
if users.get_current_user():
url = users.create_logout_url(self.request.uri)
url_linktext = 'Logout'
urla = '/'
url_admin = ""
if users.is_current_user_admin():
url = users.create_logout_url(self.request.uri)
urla = "_ah/admin/"
url_admin = 'Go to admin pages'
url_linktext = 'Logout'
else:
url = users.create_login_url(self.request.uri)
url_linktext = 'Login'
template_values = {
'url': url,
'url_linktext': url_linktext,
'url_admin': url_admin,
'urla': urla,
}
template = jinja_environment.get_template('index.html')
self.response.out.write(template.render(template_values))
class sendFollowUp(webapp2.RequestHandler):
""" Queries Datastore for fu dates that match today's date, then adds them to a task queue """
def get(self):
now = datetime.datetime.now()
now_dt = now.date() #today's date to compare with fu dates
q = emailJobs.all()
q.filter('fuperiod >', 0)
q.filter('fuperiod <', 99)
for part in q:
guid = str(part.recipientid_po)
lang = str(part.recipientlang)
trigid = str(part.triggerid)
if part.fuperiod == 1:
fu1rawdt = part.fu1
fu1dt = fu1rawdt.date()
if fu1dt == now_dt:
follow_up = '1'
if part.fuperiod == 2:
fu2rawdt = part.fu2
fu2dt = fu2rawdt.date()
if fu2dt == now_dt:
follow_up = '2'
if follow_up != None:
taskqueue.add(queue_name='emailworker', url='/emailworker', params={'guid': guid,
'fu': follow_up,
'lang': lang,
'trigid': trigid,
})
self.redirect('/emailworker')
class pushQueue(webapp2.RequestHandler):
""" Sends fu emails, updates the Datastore with datetime sent """
def store_emails(self, trigid, senddate):
db.run_in_transaction(emailJobs.update_fusent, trigid, senddate)
def get(self):
fu_messages = {'1': 'MS_x01',
'2': 'MS_x02',
}
langs = {'EN': 'English subject',
'ES': 'Spanish subject',
}
fu = str(self.request.get('fu'))
messageid = fu_messages[fu]
lang = str(self.request.get('lang'))
subject = langs[lang]
now = datetime.datetime.now()
senddate = timeStampFM(now)
guid = str(self.request.get('guid'))
trigid = str(self.request.get('trigid'))
data = {}
data['Subject'] = subject
data['MessageID'] = messageid
data['SendDate'] = senddate
data['RecipientID'] = guid
# Here I do something with data = {}
self.store_emails(trigid, senddate)
app = webapp2.WSGIApplication([('/', MainPage),
('/cron/sendfu', sendFollowUp),
('/emailworker', pushQueue)],
debug=True)
When I test the cron job at: localhost:8086/cron/sendfu
It redirects to: localhost:8086/emailworker
and I get the following error message:
Internal Server Error
The server has either erred or is incapable of performing the requested operation.
Traceback (most recent call last):
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1535, in __call__
rv = self.handle_exception(request, response, e)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1529, in __call__
rv = self.router.dispatch(request, response)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "/Users/me/Documents/workspace/gae-project/src/main.py", line 478, in get
messageid = fu_messages[fu]
KeyError: ''
from the logs:
INFO 2013-03-05 03:03:22,337 dev_appserver.py:3104] "GET /cron/sendfu HTTP/1.1" 302 -
ERROR 2013-03-05 03:03:22,348 webapp2.py:1552] ''
Traceback (most recent call last):
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1535, in __call__
rv = self.handle_exception(request, response, e)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1529, in __call__
rv = self.router.dispatch(request, response)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "/Users/me/Documents/workspace/gae-project/src/main.py", line 478, in get
messageid = fu_messages[fu]
KeyError: ''
INFO 2013-03-05 03:03:22,355 dev_appserver.py:3104] "GET /emailworker HTTP/1.1" 500 -
INFO 2013-03-05 03:03:22,509 dev_appserver.py:3104] "GET /favicon.ico HTTP/1.1" 404 -
lines:
469 def get(self):
470 fu_messages = {'1': 'MS_x01',
471 '2': 'MS_x02',
472 }
473 langs = {'EN': 'English subject',
474 'ES': 'Spanish subject',
475 }
476
477 fu = str(self.request.get('fu'))
478 messageid = fu_messages[fu]

Answer 1

When you call
fu = str(self.request.get('fu'))
if there is no 'fu' in the request, self.request.get will return the empty string (''). So when you try
messageid = fu_messages[fu]
it looks up the empty string in
fu_messages = {'1': 'MS_x01',
'2': 'MS_x02',
}
which only has '1' and '2' as keys.
The reason your pushQueue handler is not seeing the params you send via
params = {
'guid': guid,
'fu': follow_up,
'lang': lang,
'trigid': trigid,
}
taskqueue.add(queue_name='emailworker', url='/emailworker',
params=params)
is because you are using a GET handler instead of a POST or PUT handler. As the documentation states:
Params are encoded as application/x-www-form-urlencoded and set to
the payload.
So the payload of the request has your 'fu' param in it, but since it is a GET request, the payload is dropped (this is how HTTP works, not specific to App Engine). If you use POST as your handler, the payload will come through as expected.
I noticed your code is very similar to the documented sample, but simply uses get where the sample uses post.