Problem with Django-dbindexer - google-app-engine

I asked my question in appropriate Google Group (http://groups.google.com/group/django-non-relational/browse_thread/thread/a51c1903af175e1c), but developers seems are kinda busy now, so I'm afraid my question will remain unanswered. Hope I'll find solution of my problem here.
Essense is:
I use Django-nonrel + GAE + Blog App from Django Basic apps
From admin panel of my app I tried to create new blog post. And got the following exception:
DatabaseError: Lookup type 'month' isn't supported
I asked about this in related google group and was answered to use django-dbindexer.
Regarding it desription it is exactly what I need, so I guided through all instructions and "attached" it to my app. Indexing is done, but I however get the same exception. Here is the full trace:
Traceback (most recent call last):
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\core\handlers\base.py", line 111, in get_response
response = callback(request, *callback_args, **callback_kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\contrib\admin\options.py", line 308, in wrapper
return self.admin_site.admin_view(view)(*args, **kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\utils\decorators.py", line 93, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\views\decorators\cache.py", line 79, in _wrapped_view_func
response = view_func(request, *args, **kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\contrib\admin\sites.py", line 190, in inner
return view(request, *args, **kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\utils\decorators.py", line 28, in _wrapper
return bound_func(*args, **kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\utils\decorators.py", line 93, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\utils\decorators.py", line 24, in bound_func
return func(self, *args2, **kwargs2)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\db\transaction.py", line 282, in inner
res = func(*args, **kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\contrib\admin\options.py", line 852, in add_view
if form.is_valid():
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\forms\forms.py", line 121, in is_valid
return self.is_bound and not bool(self.errors)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\forms\forms.py", line 112, in _get_errors
self.full_clean()
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\forms\forms.py", line 269, in full_clean
self._post_clean()
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\forms\models.py", line 338, in _post_clean
self.validate_unique()
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\forms\models.py", line 347, in validate_unique
self.instance.validate_unique(exclude=exclude)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\db\models\base.py", line 669, in validate_unique
date_errors = self._perform_date_checks(date_checks)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\db\models\base.py", line 791, in _perform_date_checks
if qs.exists():
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\db\models\query.py", line 498, in exists
return self.query.has_results(using=self.db)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\django\db\models\sql\query.py", line 428, in has_results
return compiler.has_results()
File "F:\Its_mine19\Programming\Java\pineapplemon\src\djangotoolbox\db\basecompiler.py", line 222, in has_results
return self.get_count(check_exists=True)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\djangotoolbox\db\basecompiler.py", line 269, in get_count
return self.build_query().count(high_mark)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\djangotoolbox\db\basecompiler.py", line 275, in build_query
query.add_filters(self.query.where)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\djangotoolbox\db\basecompiler.py", line 72, in add_filters
self.add_filters(child)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\djangotoolbox\db\basecompiler.py", line 76, in add_filters
self.add_filter(column, lookup_type, self._negated, db_type, value)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\djangoappengine\db\compiler.py", line 57, in _func
return func(*args, **kwargs)
File "F:\Its_mine19\Programming\Java\pineapplemon\src\djangoappengine\db\compiler.py", line 211, in add_filter
raise DatabaseError("Lookup type %r isn't supported" % lookup_type)
DatabaseError: Lookup type 'month' isn't supported
Here is my data model (from basic.blog.models):
class Post(models.Model):
"""Post model."""
STATUS_CHOICES = (
(1, _('Draft')),
(2, _('Public')),
)
title = models.CharField(_('title'), max_length=200)
slug = models.SlugField(_('slug'), unique_for_date='publish')
author = models.ForeignKey(User, blank=True, null=True)
body = models.TextField(_('body'), )
tease = models.TextField(_('tease'), blank=True, help_text=_('Concise text suggested. Does not appear in RSS feed.'))
status = models.IntegerField(_('status'), choices=STATUS_CHOICES, default=2)
allow_comments = models.BooleanField(_('allow comments'), default=True)
publish = models.DateTimeField(_('publish'), default=datetime.datetime.now)
created = models.DateTimeField(_('created'), auto_now_add=True)
modified = models.DateTimeField(_('modified'), auto_now=True)
categories = models.ManyToManyField(Category, blank=True)
tags = TagField()
objects = PublicManager()
class Meta:
verbose_name = _('post')
verbose_name_plural = _('posts')
db_table = 'blog_posts'
ordering = ('-publish',)
get_latest_by = 'publish'
def __unicode__(self):
return u'%s' % self.title
#permalink
def get_absolute_url(self):
return ('blog_detail', None, {
'year': self.publish.year,
'month': self.publish.strftime('%b').lower(),
'day': self.publish.day,
'slug': self.slug
})
def get_previous_post(self):
return self.get_previous_by_publish(status__gte=2)
def get_next_post(self):
return self.get_next_by_publish(status__gte=2)
Here is what I have in dbindexes module of my app (basic.blog.dbindexes):
from basic.blog.models import Post
from dbindexer.api import register_index
register_index(Post, {
'publish': 'month',
# 'created': 'month',
# 'modified': 'month',
})
import logging
logger = logging.getLogger(__name__)
logger.info('Basic.blog dbindexes') # Logging statement to check if this module is ever executed to be sure indexing is done
Thanks in advance for any help.

Related

I am facing a Flask error (Attribute error: init)

I am new to flask and I am having some problems one of them being,
When I try to start running the below app on the server I get an error saying Attribute error: init
I have no clue what to do next even after a lot of googling.
from flask import *
from models import db, EmployeeModel
app = Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///theemp.db"
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
db.init_app(app)
#Linking instance from Models and creating a DB file before the user accesses the server
#app.before_first_request
def create_table():
db.create_all()
#Create view
#app.route("/data/create", methods=["GET", "POST"])
def create():
if request.method == "GET":
return render_template("create.html")
if request.method == "POST":
employee_id = request.form["employee_id"]
name = request.form["name"]
age = request.form["age"]
position = request.form["position"]
employee = EmployeeModel(employee_id=employee_id, name=name, age=age, position=position)
db.session.add(employee)
db.commit()
return redirect(url_for("data"))
if __name__=="__main__":
app.run(debug=True)
Here is the exception I get.
Traceback (most recent call last):
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/bin/flask", line 8, in <module>
sys.exit(main())
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/flask/cli.py", line 1047, in main
cli.main()
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/click/core.py", line 1055, in main
rv = self.invoke(ctx)
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/click/decorators.py", line 84, in new_func
return ctx.invoke(f, obj, *args, **kwargs)
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/flask/cli.py", line 911, in run_command
raise e from None
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/flask/cli.py", line 897, in run_command
app = info.load_app()
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/flask/cli.py", line 308, in load_app
app = locate_app(import_name, name)
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/flask/cli.py", line 218, in locate_app
__import__(module_name)
File "/home/detective/FlaskProjects/EmployeeCRUD/myapp.py", line 7, in <module>
db.init(app)
File "/home/detective/FlaskProjects/EmployeeCRUD/emp/lib/python3.10/site-packages/flask_sqlalchemy/extension.py", line 988, in __getattr__
raise AttributeError(name)
AttributeError: init

When trying to create an account number, I get an IntegrityError: FOREIGN KEY constraint failed

I want that when creating an account number, the owner of the account be the current logged-in user.
I have read about foreign key constraints but do not know what constraint the CustomUser class is having to conflict the creation of an account number.
I have tried setting on_delete to CASCADE but still it throws the IntegrityError
Please help. Thanks.
My models.py
class CustomUser(TrackingModel, AbstractUser, PermissionsMixin):
GENDER = (
("F", "Female"),
("M", "Male"),
("NS", "Not Specified"),
)
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
phone = models.CharField(max_length=20)
gender = models.CharField(max_length=6, choices=GENDER)
USERNAME_FIELD = "username"
REQUIRED_FIELDS = [
"phone",
]
#property
def token(self):
token = jwt.encode(
{
"username": self.username,
"email": self.email,
"exp": datetime.utcnow() + timedelta(hours=24),
},
settings.SECRET_KEY,
algorithm="HS256",
)
return token
def __str__(self):
return self.username
class Accountnumber(TrackingModel):
TYPE = (
("momo", "MTN Mobile Money"),
("om", "Orange Money"),
("visa", "Visa"),
("paypal", "Paypal"),
)
number = models.AutoField(primary_key=True, editable=False, default=9999)
type = models.CharField(
max_length=10,
choices=TYPE,
null=True,
default="momo",
)
customuser = models.ForeignKey(
CustomUser,
related_name="acc_no_user",
on_delete=models.CASCADE,
)
cdate = models.DateTimeField(auto_now_add=True)
enddate = models.DateTimeField(auto_now_add=True)
transactions = models.ForeignKey(
"Transaction",
related_name="acc_no_transactions",
on_delete=models.CASCADE,
default="2",
)
balance = models.PositiveIntegerField(
null=True,
blank=True,
default=00,
)
def __str__(self):
return self.number
My views.py:
class CreateAccountnumberAPIView(CreateAPIView):
serializer_class = AccountnumberSerializer
permission_classes = {
IsAuthenticated,
}
def perform_create(self, serializer):
return serializer.save(user=self.request.user)
My serializers.py:
class AccountnumberSerializer(serializers.ModelSerializer):
class Meta:
model = Accountnumber
fields = (
"number",
"type",
"cdate",
"enddate",
"transactions",
"balance",
)
My stacktrace shows at perform_create() at views.py:
Internal Server Error: /api/v1/wallet/v1/account
Traceback (most recent call last):
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\utils.py", line 98, in inner
return func(*args, **kwargs)
sqlite3.IntegrityError: FOREIGN KEY constraint failed
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\core\handlers\exception.py", line 55, in inner
response = get_response(request)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\core\handlers\base.py", line 197, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\views\decorators\csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\views\generic\base.py", line 84, in view
return self.dispatch(request, *args, **kwargs)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\rest_framework\views.py", line 509, in dispatch
response = self.handle_exception(exc)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\rest_framework\views.py", line 469, in handle_exception
self.raise_uncaught_exception(exc)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\rest_framework\views.py", line 480, in raise_uncaught_exception
raise exc
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\rest_framework\views.py", line 506, in dispatch
response = handler(request, *args, **kwargs)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\rest_framework\generics.py", line 190, in post
return self.create(request, *args, **kwargs)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\rest_framework\mixins.py", line 19, in create
self.perform_create(serializer)
File "C:\Users\USER\Documents\Dev\API\grand\wallet\views.py", line 69, in perform_create
return serializer.save(customuser=self.request.user)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\rest_framework\serializers.py", line
212, in save
self.instance = self.create(validated_data)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\rest_framework\serializers.py", line
962, in create
instance = ModelClass._default_manager.create(**validated_data)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(),
name)(*args, **kwargs)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\query.py", line 514, in create
obj.save(force_insert=True, using=self.db)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\base.py", line 806,
in save
self.save_base(
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\base.py", line 857,
in save_base
updated = self._save_table(
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\base.py", line 1000, in _save_table
results = self._do_insert(
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\base.py", line 1041, in _do_insert
return manager._insert(
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(),
name)(*args, **kwargs)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\query.py", line 1434, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\models\sql\compiler.py", line 1632, in execute_sql
self.connection.ops.fetch_returned_insert_columns(
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\backends\base\operations.py", line 208, in fetch_returned_insert_columns
return cursor.fetchone()
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\utils.py", line 97, in inner
with self:
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\utils.py", line 91, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\USER\Documents\Dev\API\grand\.venv\lib\site-packages\django\db\utils.py", line 98, in inner
return func(*args, **kwargs)
django.db.utils.IntegrityError: FOREIGN
KEY constraint failed
[13/Jan/2023 04:19:51] "POST /api/v1/wallet/v1/account HTTP/1.1" 500 183218
In your perform_create function you do the following:
def perform_create(self, serializer):
return serializer.save(user=self.request.user)
So you're saving the user variable to the user that made this request. That is fine, however, in your Accountnumber model, you store the user in the customuser field, not the user field.
Try to set your perform_create function to the following:
def perform_create(self, serializer):
return serializer.save(customuser=self.request.user)
All through, comments from my search for the answer have been that the IntegrityError of ForeignKey constriant failed mostly arises from the fact that you are trying to use a foreign key that does not yet exist and it was my case.
When creating an account number, I asked in my models.py that the field transactions be linked to the Transaction table foreign key WHICH DID NOT EXIST YET.
I, therefore, replaced the transaction field from referring to a ForeignKey to making it a CharField where I used the choices option and it worked.
Thanks, #Pavel Vergeev for editing my code
Thanks, #Ridape for pointing out an error which turns out was not the reason for the IntegrityError I was facing.

Wagtail localize 'NoneType' object has no attribute 'block'

When trying to translate my homepage with wagtail-localize, I get the error:
AttributeError at /admin/localize/submit/page/3/
'NoneType' object has no attribute 'block'
I copied my homepage and tested. I noted that some embed blocks were blank and added a value. I could translate the copy. However when I did the same on the homepage, I had the same error message.
I then deleted fields from my homepage to check which is the problematic entry. this message keeps appearing even when my homepage is empty and I try to translate only the current page (i.e. without the children)
I also deleted my previous translated homepage and recreated and new one. The error persists.
I don't really get what to do next and would highly appreciate any input on the issue.
the traceback prints:
Environment:
Request Method: POST
Request URL: https://www.fommes.org/admin/localize/submit/page/3/
Django Version: 4.0.5
Python Version: 3.9.5
Installed Applications:
['search',
'cms',
'fictionary',
'streams',
'site_settings',
'mangos',
'wagtail_localize',
'wagtail_localize.locales',
'wagtail.contrib.forms',
'wagtail.contrib.redirects',
'wagtail.embeds',
'wagtail.sites',
'wagtail.users',
'wagtail.snippets',
'wagtail.documents',
'wagtail.images',
'wagtail.search',
'wagtail.admin',
'wagtail',
'wagtail.contrib.routable_page',
'wagtail.contrib.settings',
'wagtail.contrib.table_block',
'wagtail_link_block',
'wagtailmarkdown',
'wagtail.contrib.sitemaps',
'django.contrib.sitemaps',
'modelcluster',
'taggit',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.sites',
'userauth',
'django_countries',
'allauth',
'allauth.account',
'allauth.socialaccount',
'django_comments_xtd',
'django_comments',
'widget_tweaks',
'django_extensions',
'compressor',
'dbbackup',
'wagtailcodeblock',
'captcha',
'wagtailcaptcha',
'wagtailfontawesome',
'django_jsonfield_backport']
Installed Middleware:
['django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.middleware.locale.LocaleMiddleware',
'wagtail.contrib.redirects.middleware.RedirectMiddleware']
Traceback (most recent call last):
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/django/core/handlers/exception.py", line 55, in inner
response = get_response(request)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/django/core/handlers/base.py", line 197, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/django/views/decorators/cache.py", line 62, in _wrapped_view_func
response = view_func(request, *args, **kwargs)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail/admin/urls/__init__.py", line 161, in wrapper
return view_func(request, *args, **kwargs)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail/admin/auth.py", line 182, in decorated_view
response = view_func(request, *args, **kwargs)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/django/views/generic/base.py", line 84, in view
return self.dispatch(request, *args, **kwargs)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/views/submit_translations.py", line 175, in dispatch
return super().dispatch(request, *args, **kwargs)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/django/views/generic/base.py", line 119, in dispatch
return handler(request, *args, **kwargs)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/views/submit_translations.py", line 119, in post
return self.form_valid(form)
File "/usr/local/lib/python3.9/contextlib.py", line 79, in inner
return func(*args, **kwds)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/views/submit_translations.py", line 126, in form_valid
translate_object(
File "/usr/local/lib/python3.9/contextlib.py", line 79, in inner
return func(*args, **kwds)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/operations.py", line 90, in translate_object
translator.create_translations(instance)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/operations.py", line 79, in create_translations
translation.save_target(user=self.user, publish=publish)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/models.py", line 1334, in save_target
self.source.create_or_update_translation(
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/models.py", line 763, in create_or_update_translation
ingest_segments(original, translation, self.locale, locale, segments)
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/segments/ingest.py", line 268, in ingest_segments
StreamFieldSegmentsWriter(
File "/home/teki/.virtualenvs/fommenv/lib/python3.9/site-packages/wagtail_localize/segments/ingest.py", line 235, in handle_stream_block
block.value = self.handle_block(block.block, block.value, segments)
Exception Type: AttributeError at /admin/localize/submit/page/3/
Exception Value: 'NoneType' object has no attribute 'block'

I cann't creat new table in sqlite3 from django

When I added new string in models.py his tables did't create in sqlite3. What I do wrong, again :)
python manage.py makemigrations
python manage.py migrate - I did!
It's happend, when I don't input templates information after messege:
Please enter the default value now as valid Python.
The datetime and django.utils.timezone module are available so you can do e.g. timezone.now.
I imputed just 1 and enter. After thet it broke.
If you need different information give me know, please.
Environment:
Request Method: POST
Request URL: http://127.0.0.1:8000/admin/meetups/meetup/add/
Django Version: 3.2.8
Python Version: 3.10.0
Installed Applications:
['django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'meetups']
Installed Middleware:
['django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware']
Traceback (most recent call last):
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\sqlite3\base.py", line 423, in execute
return Database.Cursor.execute(self, query, params)
The above exception (table meetups_meetup has no column named enormus) was the direct cause of the following exception:
File "C:\install\Projects_1\env\lib\site-packages\django\core\handlers\exception.py", line 47, in inner
response = get_response(request)
File "C:\install\Projects_1\env\lib\site-packages\django\core\handlers\base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\contrib\admin\options.py", line 616, in wrapper
return self.admin_site.admin_view(view)(*args, **kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\utils\decorators.py", line 130, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\views\decorators\cache.py", line 44, in _wrapped_view_func
response = view_func(request, *args, **kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\contrib\admin\sites.py", line 232, in inner
return view(request, *args, **kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\contrib\admin\options.py", line 1657, in add_view
return self.changeform_view(request, None, form_url, extra_context)
File "C:\install\Projects_1\env\lib\site-packages\django\utils\decorators.py", line 43, in _wrapper
return bound_method(*args, **kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\utils\decorators.py", line 130, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\contrib\admin\options.py", line 1540, in changeform_view
return self._changeform_view(request, object_id, form_url, extra_context)
File "C:\install\Projects_1\env\lib\site-packages\django\contrib\admin\options.py", line 1586, in _changeform_view
self.save_model(request, new_object, form, not add)
File "C:\install\Projects_1\env\lib\site-packages\django\contrib\admin\options.py", line 1099, in save_model
obj.save()
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\base.py", line 726, in save
self.save_base(using=using, force_insert=force_insert,
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\base.py", line 763, in save_base
updated = self._save_table(
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\base.py", line 868, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\base.py", line 906, in _do_insert
return manager._insert(
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\query.py", line 1270, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\sql\compiler.py", line 1416, in execute_sql
cursor.execute(sql, params)
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\utils.py", line 98, in execute
return super().execute(sql, params)
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\utils.py", line 66, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\utils.py", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\utils.py", line 79, in _execute
with self.db.wrap_database_errors:
File "C:\install\Projects_1\env\lib\site-packages\django\db\utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\sqlite3\base.py", line 423, in execute
return Database.Cursor.execute(self, query, params)
Exception Type: OperationalError at /admin/meetups/meetup/add/
Exception Value: table meetups_meetup has no column named enormus
Errors, when I input: makemigration and migrate
(env) PS C:\install\Projects_1> python manage.py makemigrations
System check identified some issues:
WARNINGS:
meetups.Meetup.participant: (fields.W340) null has no effect on ManyToManyField.
You are trying to add a non-nullable field 'dates' to meetup without a default; we can't do that (the database needs something to populate existing rows).
Please select a fix:
1) Provide a one-off default now (will be set on all existing rows with a null value for this column)
2) Quit, and let me add a default in models.py
Select an option: 1
Please enter the default value now, as valid Python
The datetime and django.utils.timezone modules are available, so you can do e.g. timezone.now
Type 'exit' to exit this prompt
>>> '2021-10-10'
You are trying to add a non-nullable field 'organizer_emails' to meetup without a default; we can't do that (the database needs something to populate existing rows).
Please select a fix:
1) Provide a one-off default now (will be set on all existing rows with a null value for this column)
2) Quit, and let me add a default in models.py
Select an option: 1
The datetime and django.utils.timezone modules are available, so you can do e.g. timezone.now
Type 'exit' to exit this prompt
>>> test#.test.com
Invalid input: invalid syntax (<string>, line 1)
>>> 'test#.test.com'
Migrations for 'meetups':
meetups\migrations\0011_auto_20211026_2116.py
- Remove field enormus from meetup
- Add field dates to meetup
- Add field organizer_emails to meetup
(env) PS C:\install\Projects_1> python manage.py migrate
System check identified some issues:
WARNINGS:
meetups.Meetup.participant: (fields.W340) null has no effect on ManyToManyField.
Operations to perform:
Apply all migrations: admin, auth, contenttypes, meetups, sessions
Running migrations:
Applying meetups.0005_auto_20211026_1234...Traceback (most recent call last):
File "C:\install\Projects_1\manage.py", line 22, in <module>
main()
File "C:\install\Projects_1\manage.py", line 18, in main
execute_from_command_line(sys.argv)
File "C:\install\Projects_1\env\lib\site-packages\django\core\management\__init__.py", line 419, in execute_from_command_line
utility.execute()
File "C:\install\Projects_1\env\lib\site-packages\django\core\management\__init__.py", line 413, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "C:\install\Projects_1\env\lib\site-packages\django\core\management\base.py", line 354, in run_from_argv
self.execute(*args, **cmd_options)
File "C:\install\Projects_1\env\lib\site-packages\django\core\management\base.py", line 398, in execute
output = self.handle(*args, **options)
File "C:\install\Projects_1\env\lib\site-packages\django\core\management\base.py", line 89, in wrapped
res = handle_func(*args, **kwargs)
File "C:\install\Projects_1\env\lib\site-packages\django\core\management\commands\migrate.py", line 244, in handle
post_migrate_state = executor.migrate(
File "C:\install\Projects_1\env\lib\site-packages\django\db\migrations\executor.py", line 117, in migrate
state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)
File "C:\install\Projects_1\env\lib\site-packages\django\db\migrations\executor.py", line 147, in _migrate_all_forwards
state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)
File "C:\install\Projects_1\env\lib\site-packages\django\db\migrations\executor.py", line 227, in apply_migration
state = migration.apply(state, schema_editor)
File "C:\install\Projects_1\env\lib\site-packages\django\db\migrations\migration.py", line 126, in apply
operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
File "C:\install\Projects_1\env\lib\site-packages\django\db\migrations\operations\fields.py", line 104, in database_forwards
schema_editor.add_field(
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\sqlite3\schema.py", line 330, in add_field
self._remake_table(model, create_field=field)
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\sqlite3\schema.py", line 191, in _remake_table
self.effective_default(create_field)
File "C:\install\Projects_1\env\lib\site-packages\django\db\backends\base\schema.py", line 324, in effective_default
return field.get_db_prep_save(self._effective_default(field), self.connection)
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\fields\__init__.py", line 842, in get_db_prep_save
return self.get_db_prep_value(value, connection=connection, prepared=False)
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\fields\__init__.py", line 1271, in get_db_prep_value
value = self.get_prep_value(value)
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\fields\__init__.py", line 1266, in get_prep_value
return self.to_python(value)
File "C:\install\Projects_1\env\lib\site-packages\django\db\models\fields\__init__.py", line 1228, in to_python
parsed = parse_date(value)
File "C:\install\Projects_1\env\lib\site-packages\django\utils\dateparse.py", line 75, in parse_date
match = date_re.match(value)
TypeError: expected string or bytes-like object

DeadlineExceededError despite using ndb.put_multi_async

I have the following POST method in my api. After sending a big JSON collection around 350 entries to the api, I get an DeadlineExceedError exception.
I already am utilising the ndb.multi_put_async. I don't know what else I could do to speed this up?
def post(self):
arguments = self.reqparser.parse_args()
json_records = arguments.get('records')
user = User.query(User.email == request.authorization.username).get()
if user:
records_put_list = []
events_put_list = []
for json_record in json_records:
record_id = json_record['record_id']
rec = Record.get_or_insert(record_id,
user=user.key,
record_date=record_date,
timestamp=record_timestamp)
for json_event in json_record['events']:
event = Event.get_or_insert(json_event['event_id'],
parent=rec.key,
user=user.key,
is_deleted=json_event['is_deleted'])
if event.timestamp < json_event['timestamp']:
event.user = user.key
event.record = rec.key
event.date_time = event_dt
event.timestamp = json_event['timestamp']
event.parent = rec.key
events_put_list.append(event)
ndb.put_multi_async(records_put_list)
ndb.put_multi_async(events_put_list)
return '', 201
else:
return '', 401
This is the exception. Any advice what I could do?
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/runtime/wsgi.py", line 266, in Handle
result = handler(dict(self._environ), self._StartResponse)
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/lib/flask/app.py", line 1836, in __call__
return self.wsgi_app(environ, start_response)
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/lib/flask/app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/lib/flask/app.py", line 1475, in full_dispatch_request
rv = self.dispatch_request()
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/lib/flask/app.py", line 1461, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/lib/flask_restful/__init__.py", line 397, in wrapper
resp = resource(*args, **kwargs)
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/lib/flask/views.py", line 84, in view
return self.dispatch_request(*args, **kwargs)
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/lib/flask_restful/__init__.py", line 487, in dispatch_request
resp = meth(*args, **kwargs)
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/application/http_basic_auth.py", line 36, in decorated
return f(*args, **kwargs)
File "/base/data/home/apps/s~feeltracker1/1-0-1-0.376950929493492366/application/rest_api_view.py", line 362, in post
is_deleted=json_event['is_deleted'])
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/model.py", line 3401, in _get_or_insert
return cls._get_or_insert_async(*args, **kwds).get_result()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 325, in get_result
self.check_success()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 320, in check_success
self.wait()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 304, in wait
if not ev.run1():
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/eventloop.py", line 235, in run1
delay = self.run0()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/eventloop.py", line 197, in run0
callback(*args, **kwds)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 474, in _on_future_completion
self._help_tasklet_along(ns, ds_conn, gen, val)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 371, in _help_tasklet_along
value = gen.send(val)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 1015, in transaction
ok = yield tconn.async_commit(options)
DeadlineExceededError
You are not using the right logic. Do not put the get_or_insert in the loop - that is never going to work for anything except the smallest number of records (and is generally an anti-pattern to avoid). Instead create the entities and append them to the list inside the loop. Once this is done, the put_multi will then take this list and handle all the puts in parallel. There is a question about how many items in a put_multi list is tolerable, but 350 is far from what I remember as a limit.
You can use a Task API to split this processing job into smaller batches (e.g. 100 entries per task - you can find the optimum number).

Resources