Why "InvalidRequest" on 2nd hit of page 2 of pagination? - google-app-engine

Question

I do pagination and a link that works to page 2 looks like this
/q?query=regionID%3D4700188&cursor=False:CqoECuYBCtgB_wDAQM8HgAD_AP8A_wD__wDAQCQA_wD_AP8A_wD_AP__AMBAHAD_AP8A_wD_AP8A___-9wllzNf_Z_-_jIGSkJGLnpCPjZCVmpyL_wB0baCgmYuMoKD_AF2ej4-akZiWkZr_AHN0bZaRm5qH_wBdnpuM_wBzdG2bkJyglpv_AF2emMaFmZLOiZ2RrZedzL2Gnc2Pk6bMrYa8iIysvpS5lLi0nsrOiLKy_wBzf56YxoWZks6JnZGtl53MvYadzY-TpsythryIjKy-lLmUuLSeys6IsrL_AP_-EDIh4o_3av1KJSkSDURvY3VtZW50SW5kZXganwEoQU5EIChJUyAiY3VzdG9tZXJfbmFtZSIgImFwcGVuZ2luZSIpIChJUyAiZ3JvdXBfbmFtZSIgInN-bW9udGFvcHJvamVjdCIpIChJUyAibmFtZXNwYWNlIiAiIikgKElTICJpbmRleF9uYW1lIiAiYWRzIikgKEVRIDQ3MDAxODguMDAwMDAwMDAwMDAgInNuYnJfcmVnaW9uSUQiKSk6GwoOKE4gc2RhdGVfZGF0ZSkQARkAAABzFPtzQjoaCg0oTiBzbmJyX2hvdXIpEAEZAAAAAAAAEEA6HAoPKE4gc25icl9taW51dGUpEAEZAAAAAAAAMkBKHAgBOhVzdDpidGlfZ2VuZXJpY19zY29yZXJAkE5SGQoMKE4gb3JkZXJfaWQpEAEZAAAAAAAA8P8
But every now and then, seemingly random, I get error message that doesn't seem to be a timeout but an error with the query. When I hit reload I got this error, is it a timeout error?
"GET /q?query=regionID%3D4700188&cursor=False:CqoECuYBCtgB_wDAQM8HgAD_AP8A_wD__wDAQCQA_wD_AP8A_wD_AP__AMBAHAD_AP8A_wD_AP8A___-9wllzNf_Z_-_jIGSkJGLnpCPjZCVmpyL_wB0baCgmYuMoKD_AF2ej4-akZiWkZr_AHN0bZaRm5qH_wBdnpuM_wBzdG2bkJyglpv_AF2emMaFmZLOiZ2RrZedzL2Gnc2Pk6bMrYa8iIysvpS5lLi0nsrOiLKy_wBzf56YxoWZks6JnZGtl53MvYadzY-TpsythryIjKy-lLmUuLSeys6IsrL_AP_-EDIh4o_3av1KJSkSDURvY3VtZW50SW5kZXganwEoQU5EIChJUyAiY3VzdG9tZXJfbmFtZSIgImFwcGVuZ2luZSIpIChJUyAiZ3JvdXBfbmFtZSIgInN-bW9udGFvcHJvamVjdCIpIChJUyAibmFtZXNwYWNlIiAiIikgKElTICJpbmRleF9uYW1lIiAiYWRzIikgKEVRIDQ3MDAxODguMDAwMDAwMDAwMDAgInNuYnJfcmVnaW9uSUQiKSk6GwoOKE4gc2RhdGVfZGF0ZSkQARkAAABzFPtzQjoaCg0oTiBzbmJyX2hvdXIpEAEZAAAAAAAAEEA6HAoPKE4gc25icl9taW51dGUpEAEZAAAAAAAAMkBKHAgBOhVzdDpidGlfZ2VuZXJpY19zY29yZXJAkE5SGQoMKE4gb3JkZXJfaWQpEAEZAAAAAAAA8P8 HTTP/1.1" 200 8611 "http://www.koolbusiness.com/q?query=regionID%3D4700188" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36" "www.koolbusiness.com" ms=133 cpu_ms=42 cpm_usd=0.060962 app_engine_release=1.8.1 instance=00c61b117cd1983617eb4b0968a84b71874563
D 2013-07-06 06:20:37.095
query regionID=4700188
E 2013-07-06 06:20:37.120
Search failed
Traceback (most recent call last):
File "/base/data/home/apps/s~montaoproject/2013e.368581150756737282/search_demo.py", line 87, in find_documents
return index.search(query)
File "/python27_runtime/python27_lib/versions/1/google/appengine/api/search/search.py", line 2732, in search
_CheckStatus(response.status())
File "/python27_runtime/python27_lib/versions/1/google/appengine/api/search/search.py", line 413, in _CheckStatus
raise _ERROR_MAP[status.code()](status.error_detail())
InvalidRequest: Failed to execute search request "regionID=4700188"
Code
def find_documents(query_string, limit, cursor):
try:
date_desc = search.SortExpression(expression='date',
direction=search.SortExpression.DESCENDING,
default_value=datetime.now().date())
hr_desc = search.SortExpression(expression='hour',
direction=search.SortExpression.DESCENDING,
default_value=datetime.now().hour)
min_desc = search.SortExpression(expression='minute',
direction=search.SortExpression.DESCENDING,
default_value=datetime.now().minute)
# Sort up to 5000 matching results by subject in descending order
sort = search.SortOptions(expressions=[date_desc, hr_desc,
min_desc], limit=10000)
# Set query options
options = search.QueryOptions(limit=limit, cursor=cursor,
sort_options=sort, number_found_accuracy=10000) # the number of results to return
# returned_fields=['author', 'subject', 'summary'],
# snippeted_fields=['content']
query = search.Query(query_string=query_string, options=options)
index = search.Index(name=_INDEX_NAME)
# Execute the query
return index.search(query)
except search.Error:
logging.exception('Search failed')
return None
regions_in = [
('3', u'Entire India'),
('4703187', u'Andaman & Nicobar Islands'),
('4694186', u'Andhra Pradesh'),
('4699188', u'Arunachal Pradesh'),
('4692186', u'Assam'),
('4702186', u'Bihar'),
('4698185', u'Chandigarh'),
('4676188', u'Chhattisgarh'),
('4691190', u'Dadra & Nagar Haveli'),
('4704183', u'Daman & Diu'),
('4699183', u'Delhi'),
('4702187', u'Goa'),
('4691189', u'Gujarat'),
('4700186', u'Haryana'),
('4703185', u'Himachal Pradesh'),
('4694187', u'Jammu & Kashmir'),
('4699189', u'Jharkhand'),
('4701185', u'Karnataka'),
('4695189', u'Kerala'),
('4700189', u'Lakshadweep'),
('4697186', u'Madhya Pradesh'),
('4694184', u'Maharashtra'),
('4700187', u'Manipur'),
('4703186', u'Meghalaya'),
('4698184', u'Mizoram'),
('4692187', u'Nagaland'),
('4696185', u'Orissa'),
('4676189', u'Pondicherry'),
('4693185', u'Punjab'),
('4701186', u'Rajasthan'),
('4701187', u'Sikkim'),
('4701188', u'Tamil Nadu'),
('4697187', u'Tripura'),
('4699190', u'Uttaranchal'),
('4692188', u'Uttar Pradesh'),
('4700188', u'West Bengal'),
]
class RegionSearch(SearchBaseHandler):
"""Handles regional search requests."""
def get(self):
"""Handles a get request with a query."""
category = None
cityentity = None
next_cursor = None
country = ''
if self.request.host.find('hipheap') > -1: country = 'USA'
elif self.request.host.find('koolbusiness') > -1: country = 'India'
elif self.request.host.find('montao') > -1: country = 'Brasil'
number_returned = 0
regionname = None
cityname = None
regionentity = None
region = None
cursor = self.request.get('cursor')
uri = urlparse(self.request.uri)
query = ''
regionID = regionid = self.request.get('regionid', 0)
cityID = cityid = self.request.get('cityid', 0)
categoryID = categoryid = self.request.get('category', 0)
if uri.query:
query = parse_qs(uri.query)
try:
query = query['query'][0]
except KeyError, err:
query = ''
# logging.info('KeyError')
# Try find region ID and/or cityID and categoryID a.s.a.p.
if regionid or query.find('regionID') > -1:
regionID = re.sub("^regionID=(\d+).*", r'\1', query)
region = Region.get_by_id(long(regionID))
regionname = region.name
if regionid:
regionID = regionid
region = Region.get_by_id(long(regionID))
regionname = region.name
if cityid or query.find('cityID') > -1:
cityID = re.sub("^.*cityID=(\d+).*", r'\1', query)
if cityid: cityID = cityid
city = montaomodel.City.get_by_id(long(cityID))
cityID = city.key().id()
cityentity = city
cityname = city.name
region = Region.get_by_id(long(city.region.key().id()))
regionID = region.key().id()
if categoryid or query.find('category') > -1:
categoryID = re.sub("^.*category=(\d+).*", r'\1', query)
if categoryid: categoryID = categoryid
logging.debug('query %s', query)
if cursor: results = find_documents(query, 50, search.Cursor(cursor))
else: results = find_documents(query, 50, search.Cursor())
if results and results.cursor: next_cursor = results.cursor.web_safe_string
namedquery = query
query = query.replace(' and company_ad=0', ''
).replace(' and company_ad=1', ''
).replace(' and category:(6010 OR 6020 OR 6030 OR 6040 OR 6090)'
, ''
).replace(' and category:(6010 OR 6020 OR 6030 OR 6040 OR 6090)'
, ''
).replace(' and category:(1020 OR 1010 OR 1030 OR 1050 OR 1080 OR 1100 OR 1090)'
, ''
).replace(' and category:(2010 OR 2030 OR 2040 OR 2080 OR 2070)'
, ''
).replace(' and category:(3040 OR 3050 OR 3030 OR 3060)'
, ''
).replace(' and category:(4010 OR 4020 OR 4040 OR 4030 OR 4090 OR 4060 OR 4070)'
, '')
query = re.sub("regionID=\d+", '', query)
query = query.replace('category and', '')
query = query.replace('type=s', '')
query = query.replace('type=w', '')
query = query.replace('type=r', '')
query = query.replace('type=b', '')
query = query.replace('cityID and', '')
query = query.replace('and ', '')
query = query.replace(' and', '')
query = query.replace('regionID', '')
query = query.replace('=', '%3D')
namedquery = namedquery.replace('=', '%3D')
query = re.sub("cityID%3D\d+", '', query)
query = re.sub("category%3D\d+", '', query)
query = query.replace(' ', ' ')
# to do: make into a dictionary for O(1) access
if int(regionid) > 0: regionname = region_id_to_name[regionid]
#if regionID and query.find('cityID') < 1:
# region = Region.get_by_id(long(regionID))
form = SearchForm()
form.w.choices = [
('4703187', u'Andaman & Nicobar Islands'),
('4694186', u'Andhra Pradesh'),
('4699188', u'Arunachal Pradesh'),
('4692186', u'Assam'),
('4702186', u'Bihar'),
('4698185', u'Chandigarh'),
('4676188', u'Chhattisgarh'),
('4691190', u'Dadra & Nagar Haveli'),
('4704183', u'Daman & Diu'),
('4699183', u'Delhi'),
('4702187', u'Goa'),
('4691189', u'Gujarat'),
('4700186', u'Haryana'),
('4703185', u'Himachal Pradesh'),
('4694187', u'Jammu & Kashmir'),
('4699189', u'Jharkhand'),
('4701185', u'Karnataka'),
('4695189', u'Kerala'),
('4700189', u'Lakshadweep'),
('4697186', u'Madhya Pradesh'),
('4694184', u'Maharashtra'),
('4700187', u'Manipur'),
('4703186', u'Meghalaya'),
('4698184', u'Mizoram'),
('4692187', u'Nagaland'),
('4696185', u'Orissa'),
('4676189', u'Pondicherry'),
('4693185', u'Punjab'),
('4701186', u'Rajasthan'),
('4701187', u'Sikkim'),
('4701188', u'Tamil Nadu'),
('4697187', u'Tripura'),
('4699190', u'Uttaranchal'),
('4692188', u'Uttar Pradesh'),
('4700188', u'West Bengal'),
]
if region or cityentity:
# to do:use memcache
form.area.choices = [] # to do: use memcache for the list
for cityitem in City.all().filter('region =',
region.key()).order('-vieworder').order('name'
).fetch(99999):
form.area.choices.append([str(cityitem.key().id()),
cityitem.name])
if cityentity:
form.area.data = str(cityentity.key().id())
if self.request.host.find('hipheap') > -1:
if region and (str(region.key().id()), region.name) \
in form.w_us.choices:
form.w_us.choices.remove((str(region.key().id()),
region.name))
else:
if region and (str(region.key().id()), region.name) \
in form.w.choices:
form.w.choices.remove((str(region.key().id()),
region.name))
if region: regionname = region.name
if results: number_returned = len(results.results)
template_values = {
'results': results,'regionname':regionname,
'cursor': next_cursor,
'country': country,
'user': self.current_user,
'number_returned': number_returned,
'loggedin': self.logged_in,
'VERSION': VERSION,
'region': region,
'regionname': regionname,
'jobs_count': get_jobs_count(self, regionID, cityID),
'estate_count': get_estate_count(self, regionID, cityID),
'electronics_count': get_electronics_count(self, regionID,
cityID),
'home_count': get_home_count(self, regionID, cityID),
'leisure_count': get_leisure_count(self, regionID, cityID),
'vehicles_count': get_vehicles_count(self, regionID,
cityID),
'cityentity': cityentity,
'request': self.request,
'categoryID': categoryID,
'form': form,
'query': query,
'namedquery': namedquery,
'cityname': cityname,
'category': category,
}
self.render_template('q.htm', template_values)
class India(SearchBaseHandler):
def get(self):
"""Handles a get request with a query."""
regionname = None
country = 'India'
cursor = self.request.get('cursor')
region = None
if self.request.host.find('hipheap') > -1: country = 'USA'
elif self.request.host.find('koolbusiness') > -1: country = 'India'
elif self.request.host.find('montao') > -1: country = 'Brasil'
uri = urlparse(self.request.uri)
query = ''
if uri.query:
query = parse_qs(uri.query)
try:
query = query['query'][0]
except KeyError, err:
query = ''
# logging.info('KeyError')
if cursor:
results = find_documents(query, 50, search.Cursor(cursor))
else:
results = find_documents(query, 50, search.Cursor())
next_cursor = None
if results and results.cursor: next_cursor = results.cursor.web_safe_string
query = query.replace(' and company_ad=0', ''
).replace(' and company_ad=1', '')
regionname = 'Entire India'
regionID = 0
cityID = 0
form = SearchForm()
form.w.choices = region_id_to_name
template_values = {
'regions':region_id_to_name,
'form': form,
'results': results,
'cursor': next_cursor,
'region': region,
'country': country,
'number_returned': len(results.results),
'jobs_count': get_jobs_count_india(self, regionID, cityID),
'estate_count': get_estate_count_india(self, regionID,
cityID),
'electronics_count': get_electronics_count_india(self,
regionID, cityID),
'home_count': get_home_count_india(self, regionID, cityID),
'leisure_count': get_leisure_count_india(self, regionID,
cityID),
'vehicles_count': get_vehicles_count_india(self, regionID,
cityID),
'user': users.get_current_user(),
'loggedin': self.logged_in,
'region': region,
'regionname': regionname,
'city': '',
'cityentity': None,
'request': self.request,
'form': SearchForm(),
'query': query,
}
self.render_template('q.htm', template_values)
The log file where the first hit is OK and the reload fails is here.
Update
The bug is changing. Now it just didn't happen until the third hit of the 2nd page. It doesn't make sense at all not to encounter a pagination error until the third hit of the second page. This is absurd so tell me what is going on and how to investigate the trace? I changed the code to say more about the trace but python doesn't inform me what it is:
It's not only RegionSearch that's failing, it's also search for empty string on pages after page 2. Since it never happens on the first page I suspect this must have something to do with the cursorbut I've been totally unable to figure out what is wrong with the cursor. The exception doesn't tell me anything, it just says ´this is not working´and nothing about why. I changed handling but it didn't generate any more output about the error: `return
index.search(query)
except search.Error as e:
logging.exception('Search failed %s', e)`
It does occur randomöly without seeming to be a timeout and the exception does not inform about the problem, all it says is "search failed" and the query that fails, a query which works if I put it in the search field. So it doesnt make sense anyhow.

Answer 1

I can only guess that it's RegionSearch that's failing.
You'll need to handle exceptions on your queries.
See https://developers.google.com/appengine/docs/python/datastore/exceptions
The exception will tell you what the error is. Hard to guess.
EDIT:
try:
# do query
except search.PutError as e:
logging.exception('caught PutError %s', e)
except search.InternalError as e:
logging.exception('caught InternalError %s', e)
except search.DeleteError as e:
logging.exception('caught DeleteError %s', e)
except search.TransientError as e:
logging.exception('caught TransientError %s', e)
except search.InvalidRequest as e:
logging.exception('caught InvalidError %s', e)
except search.Error as e:
logging.exception('caught unknown error %s', e)

Answer 2

Try setting the SortExpression's default_value to None, that worked for me.
I was getting the same issue in my Test/QA instance however in my PROD instance worked fine, setting the default_value to None solved the problem in the Test/QA instance for good.