Django migration can detect if a field was renamed and ask you about it (instead of the old fashion delete/create)
Even if multiple fields are changed it seems to find the corresponding match. For example:
Before:
class DirectoryMirror(models.Model):
directory_origin = models.ForeignKey(TapeDirectory)
machine_target = models.ForeignKey(GenericMachine)
directory_target = models.CharField(max_length=255, blank=False)
After (changing field names):
class DirectoryMirror(models.Model):
source_directory = models.ForeignKey(TapeDirectory)
target_machine = models.ForeignKey(GenericMachine)
target_directory = models.CharField(max_length=255, blank=False)
Generating migration:
$ ./manage.py makemigrations
Did you rename directorymirror.directory_origin to directorymirror.source_directory (a ForeignKey)? [y/N] y
Did you rename directorymirror.directory_target to directorymirror.target_directory (a CharField)? [y/N] y
Did you rename directorymirror.machine_target to directorymirror.target_machine (a ForeignKey)? [y/N] y
How does it manage to detect the renaming and find the correct match?
Here it is the algorithm https://github.com/django/django/blob/bc77eb6d0858652e197c08c299efaeb06c51efee/django/db/migrations/autodetector.py#L757
Copying it here
def generate_renamed_fields(self):
"""
Works out renamed fields
"""
self.renamed_fields = {}
for app_label, model_name, field_name in sorted(self.new_field_keys - self.old_field_keys):
old_model_name = self.renamed_models.get((app_label, model_name), model_name)
old_model_state = self.from_state.models[app_label, old_model_name]
field = self.new_apps.get_model(app_label, model_name)._meta.get_field(field_name)
# Scan to see if this is actually a rename!
field_dec = self.deep_deconstruct(field)
for rem_app_label, rem_model_name, rem_field_name in sorted(self.old_field_keys - self.new_field_keys):
if rem_app_label == app_label and rem_model_name == model_name:
old_field_dec = self.deep_deconstruct(old_model_state.get_field_by_name(rem_field_name))
if field.remote_field and field.remote_field.model and 'to' in old_field_dec[2]:
old_rel_to = old_field_dec[2]['to']
if old_rel_to in self.renamed_models_rel:
old_field_dec[2]['to'] = self.renamed_models_rel[old_rel_to]
if old_field_dec == field_dec:
if self.questioner.ask_rename(model_name, rem_field_name, field_name, field):
self.add_operation(
app_label,
operations.RenameField(
model_name=model_name,
old_name=rem_field_name,
new_name=field_name,
)
)
self.old_field_keys.remove((rem_app_label, rem_model_name, rem_field_name))
self.old_field_keys.add((app_label, model_name, field_name))
self.renamed_fields[app_label, model_name, field_name] = rem_field_name
break
Related
I had a company assign me an assignment to implement a fileSystem class to run shell commands through python without using any libraries. Does anyone have any suggestions on how to get started? Not quite sure how to tackle this problem.
Problem:
Implement a FileSystem class using python
Root path is '/'.
Path separator is '/'.
Parent directory is addressable as '..'.
Directory names consist only of English alphabet letters (A-Z and a-z).
All functions should support both relative and absolute paths.
All function parameters are the minimum required/recommended parameters.
Any additional class/function can be added.
What I've worked on so far:
class Path:
def __init__(self, path):
self.current_path = path.split("/")
def cd(self, new_path):
new_split = new_path.split("/")
for i in new_split:
if i == "..":
new_split.pop(0)
self.current_path = self.current_path[:-1]
self.current_path += new_split
def getString(self):
return "/".join(self.current_path)
def pwd(self, path):
return self.current_path
def mkdir():
pass
def rmdir():
pass
#driver code
fs = Path()
fs.mkdir('usr')
fs.cd('usr')
fs.mkdir('local')
fs.cd('local')
return fs.pwd()
So, this is what I came up with. I know I need to clean it up
'''
class Path:
dir_stack = []
def __init__(self):
print("started")
main_dir = {'/': {}}
self.dir_stack.insert( len(self.dir_stack), main_dir)
def getCurrentMap():
global current_Level
current_Level = self.dir_stack[len(self.dir_stack) - 1]
def cd(self, folder):
if(folder == '../'):
self.dir_stack.pop()
current_Level = self.dir_stack[len(self.dir_stack) - 1]
current_Map = current_Level[(list(current_Level.keys())[0])]
print('lev', current_Map)
if folder in current_Map:
print('here')
self.dir_stack.insert(len(self.dir_stack), current_Map)
else:
print ("no existing folder")
def pwd(self):
path = ''
print(self.dir_stack)
for x in self.dir_stack:
path += (list(x.keys())[0]) + '/'
print(path)
def ls(self):
current_Level = self.dir_stack[len(self.dir_stack) - 1]
current_Map = current_Level[(list(current_Level.keys())[0])]
print(current_Map)
def mkdir(self, folder_Name):
current_Level = self.dir_stack[len(self.dir_stack) - 1]
newDir = {folder_Name: {}}
current_Map = current_Level[(list(current_Level.keys())[0])]
if folder_Name in current_Map:
warning = folder_Name + ' already exists in directory'
print(warning)
else:
current_Map.update(newDir)
def rmdir(self, folder_Name):
current_Level = self.dir_stack[len(self.dir_stack) - 1]
#make global var current_Map
current_Map = current_Level[(list(current_Level.keys())[0])]
if folder_Name in current_Map:
del current_Map[folder_Name]
else:
print('folder doesnt exist')
# driver code
fs = Path()
fs.mkdir('usr')
fs.mkdir('new')
fs.mkdir('files')
fs.cd('usr')
fs.mkdir('local')
fs.cd('new')
fs.pwd()
fs.cd('../')
fs.ls()
# fs.mkdir('local')
# fs.cd('local')
fs.pwd()
I'v been banging my head over this for 2 days straight and i still can understand/find a way to do this and it looks super simple but im obviously overlooking something (also im fairly new to DBs :) ).
I want to have Owner and Pet model.
Pets have 'owner ids' as foreign keys, and Owners have 'pets' as relationship, so far so good.
But now i also want Owners to have one 'pet id' written as 'favorite pet'.
Having foreign keys in both models (each others keys) started to make bunch of different problems (different depending on how i try to solve it, but either circular dependency or some multipath error)
I also noticed that if i avoid having 'favourite_pet_id'-foreign key in Owner model, keeping only favourite_pet-relationship, then i dont have this written anywhere in DB (at least not visible), it exists only as 'relationship' ?
What would be correct way of doing this ?
Thanks in advance !
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
class Owner(db.Model):
id = db.Column(db.Integer, primary_key=True)
pets = db.relationship('Pet', foreign_keys='Pet.owner_id')
favourite_id = db.Column(db.Integer, db.ForeignKey('pet.id'))
favourite = db.relationship('Pet', uselist=False, foreign_keys='Owner.favourite_id')
class Pet(db.Model):
id =db.Column(db.Integer, primary_key=True)
owner_id = db.Column(db.Integer, db.ForeignKey('owner.id'))
owner = db.relationship('Owner', uselist=False, back_populates='pets', foreign_keys='Pet.owner_id')
o = Owner() # one owner
p1 = Pet() # pet 1
p2 = Pet() # pet 2
p1.owner=o # setting owner for pet1
p2.owner=o # setting owner for pet2
o.favourite=p2 # setting pet2 to be favourite
#db.session.add(o)
#db.session.add(p1)
#db.session.add(p2)
#db.session.commit()
print (p1.owner) # owner
print (p2.owner) # owner
print (p1) # pet 1
print (p2) # pet 2
print (o.pets) # owners pets
print (o.favourite) # favourite pet
Here's a working version of your code below. The key is being more explicit about the relationship (e.g. join condition) as there are multiple relationships between the models/tables.
Note: While not part of your question/request, I also reformatted a bit for PEP8 conformity and readability. The later is a good practice to adopt early on as model files typically grow quickly and can become very difficult to read, digest, and debug.
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
class Owner(db.Model):
id = db.Column(
db.Integer,
primary_key=True,
)
favourite_pet_id = db.Column(
db.Integer,
db.ForeignKey('pet.id'),
nullable=True,
)
favourite_pet = db.relationship(
'Pet',
uselist=False,
foreign_keys=[favourite_pet_id],
primaryjoin='Pet.owner_id == Owner.favourite_pet_id',
)
class Pet(db.Model):
id = db.Column(
db.Integer,
primary_key=True,
)
owner_id = db.Column(
db.Integer,
db.ForeignKey('owner.id'),
)
owner = db.relationship(
'Owner',
uselist=False,
foreign_keys=[owner_id],
primaryjoin='Pet.owner_id == Owner.id',
backref=db.backref(
'pets',
uselist=True,
),
)
db.create_all()
o = Owner() # one owner
p1 = Pet() # pet 1
p2 = Pet() # pet 2
p1.owner = o # setting owner for pet1
p2.owner = o # setting owner for pet2
o.favourite_pet = p2 # setting pet2 to be favourite
print(p1.owner) # owner
print(p2.owner) # owner
print(p1) # pet 1
print(p2) # pet 2
print(o.pets) # owners pets
print(o.favourite) # favourite pet
class PostTemplate(BaseModel):
content = TextField(unique=True)
class VkGroup(BaseModel):
group_id = IntegerField(unique=True)
class PostTemplateVkGroup(BaseModel):
"""
http://charlesleifer.com/blog/a-tour-of-tagging-schemas-many-to-many-bitmaps-and-more/
"""
group = ForeignKeyField(VkGroup)
post_template = ForeignKeyField(PostTemplate)
def get_posted_templates_for_group(group_id: int) -> Iterable:
"""Get posted templates.
Args:
group_id (int): id группы
"""
queries = (PostTemplate
.select()
.join(PostTemplateVkGroups)
.join(VkGroup)
.where(VkGroup.group_id == group_id))
return queries
all_post_templates = PostTemplate.select()
Many-to-many relationship.
For every record in PostTemplateVkGroup post template from this record is used in group from this record.
all_post_templates = not_posted_templates | posted_templates
How I can get not_posted_templates?
If your database supports the "EXCEPT" operation, you can:
all_post_templates = PostTemplate.alias().select()
post_templates = get_posted_templates_for_group(...)
difference = all_post_templates - post_templates
Sqlite example:
class Post(Base):
title = TextField()
class Tag(Base):
tag = TextField()
class PostTag(Base):
post = ForeignKeyField(Post)
tag = ForeignKeyField(Tag)
db.create_tables([Post, Tag, PostTag])
data = (
('pa', ('ta1', 'ta2')),
('pb', ('tb1', 'tb2')),
('pc', ()))
for title, tags in data:
post = Post.create(title=title)
for tag in tags:
tag = Tag.create(tag=tag)
PostTag.create(post=post, tag=tag)
# Create some tags that aren't associated with any post.
Tag.create(tag='tx1')
Tag.create(tag='tx2')
pa1_tags = (Tag
.select()
.join(PostTag)
.join(Post)
.where(Post.title == 'pa'))
all_tags = Tag.alias().select()
diff = all_tags - pa1_tags
for t in diff:
print(t.tag)
# Prints
# tb1
# tb2
# tx1
# tx2
I'm using google app engine and need to have the keys of an entity between 1000 and 2^31. I'm considering 2 ways of doing this:
1) keep a counter of the created keys as detailed here https://cloud.google.com/appengine/articles/sharding_counters. But this requires several datastore read/writes for every key and I'm not sure it is guaranteed to be consistent.
2) generate a random int in my range and check if that key is already in the database. To make it cheap, i'd like a keys_only query, but i can't find a way to do this except saving the key also as a separate field:
MyEntity.query(MyEntity.key_field==new_random_number).fetch(keys_only=True)
Is there a better way to achieve this?
How many writes per second are you expecting in production? Both of your proposals are good, but for our application I decided to go with a sharded counter approach. You can also set the id of an entity before you put it to avoid the query altogether:
MyModel(id="foo")
then you can look it up:
MyModel.get_by_id("foo")
Id doesn't have to be a string, it can be a number also:
MyModel(id=123)
If you decide to go with the sharded counter, here's our production-level code which is darn close what you read in that article ;o) Memcache adds the level of consistency we needed to be able to get the right count.
class GeneralShardedCounterConfig(ndb.Model):
SHARD_KEY_TEMPLATE = 'gen-count-{}-{:d}'
num_shards = ndb.IntegerProperty(default=200)
#classmethod
def all_keys(cls, name):
config = cls.get_or_insert(name)
shard_key_strings = [GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
for index in range(config.num_shards)]
return [ndb.Key(GeneralShardedCounter, shard_key_string)
for shard_key_string in shard_key_strings]
class GeneralShardedCounter(BaseModel):
count = ndb.IntegerProperty(default=0)
#classmethod
def get_count(cls, name):
total = memcache.get(name)
if total is None:
total = 0
all_keys = GeneralShardedCounterConfig.all_keys(name)
for counter in ndb.get_multi(all_keys):
if counter is not None:
total += counter.count
memcache.set(name, total, constants.SHORT_MEMCACHE_TTL)
return total
#classmethod
#ndb.transactional(retries=5)
def increase_shards(cls, name, num_shards):
config = GeneralShardedCounterConfig.get_or_insert(name)
if config.num_shards < num_shards:
config.num_shards = num_shards
config.put()
#classmethod
#ndb.transactional(xg=True)
def _increment(cls, name, num_shards):
index = random.randint(0, num_shards - 1)
shard_key_string = GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
counter = cls.get_by_id(shard_key_string)
if counter is None:
counter = cls(id=shard_key_string)
counter.count += 1
counter.put()
# Memcache increment does nothing if the name is not a key in memcache
memcache.incr(name)
#classmethod
def increment(cls, name):
config = GeneralShardedCounterConfig.get_or_insert(name)
cls._increment(name, config.num_shards)
#classmethod
def _add(cls, name, value, num_shards):
index = random.randint(0, num_shards - 1)
shard_key_string = GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
counter = cls.get_by_id(shard_key_string)
if counter is None:
counter = cls(id=shard_key_string)
counter.count += value
counter.put()
# Memcache increment does nothing if the name is not a key in memcache
memcache.incr(name, value)
#classmethod
def add(cls, name, value):
config = GeneralShardedCounterConfig.get_or_insert(name)
cls._add(name, value, config.num_shards)
Example of get_or_insert. Insert 7 unique keys
import webapp2
from google.appengine.ext import ndb
from datetime import datetime
import random
import logging
class Examples(ndb.Model):
data = ndb.StringProperty()
modified = ndb.DateTimeProperty(auto_now=True)
created = ndb.DateTimeProperty() # NOT auto_now_add HERE !!
class MainHandler(webapp2.RequestHandler):
def get(self):
count = 0
while count < 7:
random_key = str(random.randrange(1, 9))
dt_created = datetime.now()
example = Examples.get_or_insert(random_key, created=dt_created, data='some data for ' + random_key)
if example.created != dt_created:
logging.warning('Random key %s not unique' % random_key)
continue
count += 1
self.response.write('Keys inserted')
app = webapp2.WSGIApplication([
('/', MainHandler)
], debug=True)
I am getting this traceback;
--- Trimmed parts ---
File "C:\Users\muhammed\Desktop\gifdatabase\gifdatabase.py", line 76, in maketransaction
gif.tags = list(set(gif.tags + tags))
File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\model.py", line 2893, in __hash__
raise TypeError('Model is not immutable')
TypeError: Model is not immutable
Here is related parts of my code;
class Gif(ndb.Model):
author = ndb.UserProperty()
#tags = ndb.StringProperty(repeated=True)
tags = ndb.KeyProperty(repeated=True)
#classmethod
def get_by_tag(cls,tag_name):
return cls.query(cls.tags == ndb.Key(Tag, tag_name)).fetch()
class Tag(ndb.Model):
gif_count = ndb.IntegerProperty()
class PostGif(webapp2.RequestHandler):
def post(self):
user = users.get_current_user()
if user is None:
self.redirect(users.create_login_url("/static/submit.html"))
return
link = self.request.get('gif_link')
tag_names = shlex.split(self.request.get('tags').lower())
#ndb.transactional(xg=True)
def maketransaction():
tags = [Tag.get_or_insert(tag_name) for tag_name in tag_names]
gif = Gif.get_or_insert(link)
if not gif.author: # first time submission
gif.author = user
gif.tags = list(set(gif.tags + tags))
gif.put()
for tag in tags:
tag.gif_count += 1
tag.put()
if validate_link(link) and tag_names:
maketransaction()
self.redirect('/static/submit_successful.html')
else:
self.redirect('/static/submit_fail.html')
What is the problem with gif.tags = list(set(gif.tags + tags)) line?
You are inserting tags instead of keys, you need to access
tags = [Tag.get_or_insert(tag_name).key .....]
but you can also make this a single network hop like this
futures = [Tag.get_or_insert_async(tag_name) for tag_name in tag_names]
futures.append(Gif.get_or_insert_async(link))
ndb.Future.wait_all(futures)
gif = futures.pop().get_result()
tags = [future.get_result() for future in futures]
but that's not really the question just a suggestion ^, for clearer answer with .key is
gif.tags = gif.tags + [tag.key for tag in tags]
# or
gif.tags.extend([tag.key for tag in tags])