class Dummy(TestCase):
def setUp(self):
thing = Thing.objects.create(name="Thing")
def test_a(self):
self.assertTrue(Thing.objects.get(pk=1))
def test_b(self):
self.assertTrue(Thing.objects.get(pk=1))
In this example I expect for setUp to be run prior to every test case, but it is only run prior to the first and then the changes are rolled back. This causes test_a to pass, but the equivalent test_b to fail. Is this the expected behavior? What do I need to do to make sure that the database is in the same state prior to every test case?
Figured it out. setUp is being run each time, it's just that it's incrementing the private key in the database. Therefore the Thing with pk=1 no longer exists. This works just fine:
class Dummy_YepThatsMe(TestCase):
def setUp(self):
thing = Thing.objects.create(name="Thing")
def test_a(self):
self.assertTrue(Thing.objects.get(name="Thing"))
def test_b(self):
self.assertTrue(Thing.objects.get(name="Thing"))
Related
I want to ensure that my Class' staticmethod is called with the correct arguments without actually calling it, therefore I am mocking it. E.g.:
import unittest
from unittest.mock import patch
class FooStatic:
#staticmethod
def bar_static(self, baz_static):
print(baz_static)
pass
class TestFooStatic(unittest.TestCase):
def test_foo_static(self):
with patch.object(FooStatic, 'bar_static', autospec=True):
FooStatic.bar_static()
def test_foo_static_instance(self):
with patch.object(FooStatic, 'bar_static', autospec=True):
foo_s = FooStatic()
foo_s.bar_static()
Both these tests should complain that FooStatic.bar_static cannot be called without argument 'baz_static'. Unfortunately they don't, the tests succeed.
Without the staticmethod decorator, patch behaves as I expect:
class Foo:
def bar(self, baz):
print(baz)
pass
class TestFoo(unittest.TestCase):
def test_foo(self):
with patch.object(Foo, 'bar', autospec=True):
foo = Foo()
foo.bar() # raises TypeError: missing a required argument: 'baz'
I have found a loosely related issue in python, that was fixed from python 3.7 onwards: merged PR.
I am on python 3.8.5 (default on ubunutu 20).
I am not opposed to investing some time to try and propose a fix myself. However, I first want to make sure I am not overlooking anything. Any thoughts?
This is ridicolously trivial but i've spent half an hour trying to solve it.
class SocialPost(model.Model):
total_comments=model.IntegerProperty(default=0)
def create_reply_comment(self,content,author):
...
logging.info(self)
self.total_comments=self.total_comments+1
self.put()
In the logfile, i can see how total_comments is 0 but in the admin console, it is 1. The other fields are correct, except for this one.
Probably there's something wrong in that "default=0" but i can't find what is wrong.
Edit: full code of my function
def create_reply_comment(self,content,author):
floodControl=memcache.get("FloodControl-"+str(author.key))
if floodControl:
raise base.FloodControlException
new_comment= SocialComment(parent=self.key)
new_comment.author=author.key
new_comment.content=content
new_comment.put()
logging.info(self)
self.latest_comment_date=new_comment.creation_date
self.latest_comment=new_comment.key
self.total_comments=self.total_comments+1
self.put()
memcache.add("FloodControl-"+str(author.key), datetime.now(),time=SOCIAL_FLOOD_TIME)
Where i call the function:
if cmd == "create_reply_post":
post=memcache.get("SocialPost-"+str(self.request.get('post')))
if post is None:
post=model.Key(urlsafe=self.request.get('post')).get()
memcache.add("SocialPost-"+str(self.request.get('post')),post)
node=node.get()
if not node.get_subscription(user).can_reply:
self.success()
return
post.create_reply_comment(feedparser._sanitizeHTML(self.request.get("content"),"UTF-8"),user)
You're calling memcache.add before you make your change to total_comments, so when you read it back from memcache on subsequent calls, you're getting an out-of-date value from the cache. Your create_reply_comment needs to either delete or overwrite the "SocialPost-"+str(self.request.get('post') cache key.
[edit] Though your post title says you're using NDB (model.Model though? Hmm.), so you could just skip the memcache bits entirely, and let NDB do it's thing?
I would like to rewrite some of my tasks as pipelines. Mainly because of the fact that I need a way of detecting when a task finished or start a tasks in specific order. My problem is that I'm not sure how to rewrite the recursive tasks to pipelines. By recursive I mean tasks that call themselves like this:
class MyTask(webapp.RequestHandler):
def post(self):
cursor = self.request.get('cursor', None)
[set cursor if not null]
[fetch 100 entities form datastore]
if len(result) >= 100:
[ create the same task in the queue and pass the cursor ]
[do actual work the task was created for]
Now I would really like to write it as a pipeline and do something similar to:
class DoSomeJob(pipeline.Pipeline):
def run(self):
with pipeline.InOrder():
yield MyTask()
yield MyOtherTask()
yield DoSomeMoreWork(message2)
Any help with this one will be greatly appreciated. Thank you!
A basic pipeline just returns a value:
class MyFirstPipeline(pipeline.Pipeline):
def run(self):
return "Hello World"
The value has to be JSON serializable.
If you need to coordinate several pipelines you will need to use a generator pipeline and the yield statement.
class MyGeneratorPipeline(pipeline.Pipeline):
def run(self):
yield MyFirstPipeline()
You can treat the yielding of a pipeline as if it returns a 'future'.
You can pass this future as the input arg to another pipeline:
class MyGeneratorPipeline(pipeline.Pipeline):
def run(self):
result = yield MyFirstPipeline()
yield MyOtherPipeline(result)
The Pipeline API will ensure that the run method of MyOtherPipeline is only called once the result future from MyFirstPipeline has been resolved to a real value.
You can't mix yield and return in the same method. If you are using yield the value has to be a Pipeline instance. This can lead to a problem if you want to do this:
class MyRootPipeline(pipeline.Pipeline):
def run(self, *input_args):
results = []
for input_arg in input_args:
intermediate = yield MyFirstPipeline(input_arg)
result = yield MyOtherPipeline(intermediate)
results.append(result)
yield results
In this case the Pipeline API just sees a list in your final yield results line, so it doesn't know to resolve the futures inside it before returning and you will get an error.
They're not documented but there is a library of utility pipelines included which can help here:
https://code.google.com/p/appengine-pipeline/source/browse/trunk/src/pipeline/common.py
So a version of the above which actually works would look like:
import pipeline
from pipeline import common
class MyRootPipeline(pipeline.Pipeline):
def run(self, *input_args):
results = []
for input_arg in input_args:
intermediate = yield MyFirstPipeline(input_arg)
result = yield MyOtherPipeline(intermediate)
results.append(result)
yield common.List(*results)
Now we're ok, we're yielding a pipeline instance and Pipeline API knows to resolve its future value properly. The source of the common.List pipeline is very simple:
class List(pipeline.Pipeline):
"""Returns a list with the supplied positional arguments."""
def run(self, *args):
return list(args)
...at the point that this pipeline's run method is called the Pipeline API has resolved all of the items in the list to actual values, which can be passed in as *args.
Anyway, back to your original example, you could do something like this:
class FetchEntitites(pipeline.Pipeline):
def run(self, cursor=None)
if cursor is not None:
cursor = Cursor(urlsafe=cursor)
# I think it's ok to pass None as the cursor here, haven't confirmed
results, next_curs, more = MyModel.query().fetch_page(100,
start_cursor=cursor)
# queue up a task for the next page of results immediately
future_results = []
if more:
future_results = yield FetchEntitites(next_curs.urlsafe())
current_results = [ do some work on `results` ]
# (assumes current_results and future_results are both lists)
# this will have to wait for all of the recursive calls in
# future_results to resolve before it can resolve itself:
yield common.Extend(current_results, future_results)
Further explanation
At the start I said we can treat result = yield MyPipeline() as if it returns a 'future'. This is not strictly true, obviously we are actually just yielding the instantiated pipeline. (Needless to say our run method is now a generator function.)
The weird part of how Python's yield expressions work is that, despite what it looks like, the value that you yield goes somewhere outside the function (to the Pipeline API apparatus) rather than into your result var. The value of the result var on the left side of the expression is also pushed in from outside the function, by calling send on the generator (the generator being the run method you defined).
So by yielding an instantiated Pipeline, you are letting the Pipeline API take that instance and call its run method somewhere else at some other time (in fact it will be passed into a task queue as a class name and a set of args and kwargs and re-instantiated there... this is why your args and kwargs need to be JSON serializable too).
Meanwhile the Pipeline API sends a PipelineFuture object into your run generator and this is what appears in your result var. It seems a bit magical and counter-intuitive but this is how generators with yield expressions work.
It's taken quite a bit of head-scratching for me to work it out to this level and I welcome any clarifications or corrections on anything I got wrong.
When you create a pipeline, it hands back an object that represents a "stage". You can ask the stage for its id, then save it away. Later, you can reconstitute the stage from the saved id, then ask the stage if it's done.
See http://code.google.com/p/appengine-pipeline/wiki/GettingStarted and look for has_finalized. There's an example that does most of what you need.
in my app i for one of the handler i need to get a bunch of entities and execute a function for each one of them.
i have the keys of all the enities i need. after fetching them i need to execute 1 or 2 instance methods for each one of them and this slows my app down quite a bit. doing this for 100 entities takes around 10 seconds which is way to slow.
im trying to find a way to get the entities and execute those functions in parallel to save time but im not really sure which way is the best.
i tried the _post_get_hook but the i have a future object and need to call get_result() and execute the function in the hook which works kind of ok in the sdk but gets a lot of 'maximum recursion depth exceeded while calling a Python objec' but i can't really undestand why and the error message is not really elaborate.
is the Pipeline api or ndb.Tasklets what im searching for?
atm im going by trial and error but i would be happy if someone could lead me to the right direction.
EDIT
my code is something similar to a filesystem, every folder contains other folders and files. The path of the Collections set on another entity so to serialize a collection entity i need to get the referenced entity and get the path. On a Collection the serialized_assets() function is slower the more entities it contains. If i could execute a serialize function for each contained asset side by side it would speed things up quite a bit.
class Index(ndb.Model):
path = ndb.StringProperty()
class Folder(ndb.Model):
label = ndb.StringProperty()
index = ndb.KeyProperty()
# contents is a list of keys of contaied Folders and Files
contents = ndb.StringProperty(repeated=True)
def serialized_assets(self):
assets = ndb.get_multi(self.contents)
serialized_assets = []
for a in assets:
kind = a._get_kind()
assetdict = a.to_dict()
if kind == 'Collection':
assetdict['path'] = asset.path
# other operations ...
elif kind == 'File':
assetdict['another_prop'] = asset.another_property
# ...
serialized_assets.append(assetdict)
return serialized_assets
#property
def path(self):
return self.index.get().path
class File(ndb.Model):
filename = ndb.StringProperty()
# other properties....
#property
def another_property(self):
# compute something here
return computed_property
EDIT2:
#ndb.tasklet
def serialized_assets(self, keys=None):
assets = yield ndb.get_multi_async(keys)
raise ndb.Return([asset.serialized for asset in assets])
is this tasklet code ok?
Since most of the execution time of your functions are spent waiting for RPCs, NDB's async and tasklet support is your best bet. That's described in some detail here. The simplest usage for your requirements is probably to use the ndb.map function, like this (from the docs):
#ndb.tasklet
def callback(msg):
acct = yield ndb.get_async(msg.author)
raise tasklet.Return('On %s, %s wrote:\n%s' % (msg.when, acct.nick(), msg.body))
qry = Messages.query().order(-Message.when)
outputs = qry.map(callback, limit=20)
for output in outputs:
print output
The callback function is called for each entity returned by the query, and it can do whatever operations it needs (using _async methods and yield to do them asynchronously), returning the result when it's done. Because the callback is a tasklet, and uses yield to make the asynchronous calls, NDB can run multiple instances of it in parallel, and even batch up some operations.
The pipeline API is overkill for what you want to do. Is there any reason why you couldn't just use a taskqueue?
Use the initial request to get all of the entity keys, and then enqueue a task for each key having the task execute the 2 functions per-entity. The concurrency will be based then on the number of concurrent requests as configured for that taskqueue.
The apply phase of save may fail and/or is still being done asynchronously before next not strongly-consistent read — non ancestor query.
Based on local testing article I have wrote a test that should simulate inconsistent reads:
import dev_appserver
dev_appserver.fix_sys_path()
import unittest
from google.appengine.ext import ndb
from google.appengine.ext import testbed
from google.appengine.datastore import datastore_stub_util
class SomeModel(ndb.Model):
pass
class SingleEntityConsistency(unittest.TestCase):
def setUp(self):
# Setup AppEngine env
self.testbed = testbed.Testbed()
self.testbed.activate()
self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=0)
self.testbed.init_datastore_v3_stub(consistency_policy=self.policy)
self.testbed.init_memcache_stub()
# A test key
self.key = ndb.Key('SomeModel', 'test')
def tearDown(self):
self.testbed.deactivate()
def test_tx_get_or_insert(self):
p = SomeModel.get_or_insert('test')
self.assertEqual(0, SomeModel.query().count(1), "Shouldn't be applied yet")
self.assertEqual(1, SomeModel.query(ancestor=self.key).count(1), "Ancestor query read should be consistent")
def test_no_tx_insert(self):
p = SomeModel(id='test')
p.put()
self.assertEqual(0, SomeModel.query().count(2), "Shouldn't be applied yet")
self.assertEqual(1, SomeModel.query(ancestor=self.key).count(1), "Ancestor query read should be consistent")
def test_with_ancestor(self):
p = SomeModel(id='test')
p.put()
self.assertEqual(p, SomeModel.query(ancestor=self.key).get())
def test_key(self):
p = SomeModel(id='test')
p.put()
self.assertEqual(p, self.key.get())
if __name__ == '__main__':
unittest.main()
Actual questions…
Does wrapping put() in transaction change behaviour described in the beginning? Do I still need a strongly consistent query to make a sure that I'll read was was written in the txn? (tests suggest that, I still need strongly consistent query)
Is key.get() considered to be strongly-consistent? (tests suggest that, it is)
UPDATE
I have updated test code as Guido mentioned, now all test pass:
self.testbed.init_datastore_v3_stub(consistency_policy=self.policy)
I believe you must do something to activate the policy. That would explain the test failures. Also I believe only queries are affected and a lone put is effectively a transaction. Finally beware of NDB's caches.