Why does the following code give me an error when trying to write to a SQL Server but not on sqlite3 or mysql?
obj, created = cls.objects.using(database_data).update_or_create(**item)
The datamodel is defined below:
class fubar(models.Model)
region = models.CharField(max_length=32, default='ABCD')
market_participant = models.CharField(max_length=32, default='ACME')
trade_date = models.DateField()
interval_begin = models.DateTimeField()
interval_end = models.DateTimeField()
interval_type = models.CharField(max_length=32, default='5MIN')
location = models.CharField(max_length=32, default=None, null=True, blank=True)
location_type = models.CharField(max_length=32, default=None)
class Meta:
unique_together = [
'region',
'market_participant',
'trade_date',
'interval_begin',
'interval_end',
'interval_type',
'location',
'location_type'
]
IntegrityError: (
'23000',
"[23000] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]
Cannot insert duplicate key row in object 'dbo.foo`
with unique index 'foo_region_market_participant_trade_date_interval_begin_inte_d7d3bf7e_uniq'.
The duplicate key value is
(ABCD, ACME, 2020-12-06, 2020-12-06 00:00:00.0000000, 2020-12-06 00:05:00.0000000, 5MIN, BAZ, QUX).
(2601)
(SQLExecDirectW)"
)
Note: Error code was originally one line. I have re-formatted for readability.
Related
I am using glue job to write data pipeline. I took code from community, which is as following
import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.dynamicframe import DynamicFrame
from awsglue.job import Job
from py4j.java_gateway import java_import
SNOWFLAKE_SOURCE_NAME = "net.snowflake.spark.snowflake"
#args = getResolvedOptions(sys.argv, ['JOB_NAME'])
args = getResolvedOptions(sys.argv, ['JOB_NAME', 'URL', 'ACCOUNT', 'WAREHOUSE', 'DB', 'SCHEMA', 'USERNAME', 'PASSWORD', 'ROLE'])
sparkContext = SparkContext()
glueContext = GlueContext(sparkContext)
sparkSession = glueContext.spark_session
glueJob = Job(glueContext)
glueJob.init(args['JOB_NAME'], args)
##Use the CData JDBC driver to read Snowflake data from the Products table into a DataFrame
##Note the populated JDBC URL and driver class name
java_import(sparkSession._jvm, SNOWFLAKE_SOURCE_NAME)
sparkSession._jvm.net.snowflake.spark.snowflake.SnowflakeConnectorUtils.enablePushdownSession(sparkSession._jvm.org.apache.spark.sql.SparkSession.builder().getOrCreate())
tmp_dir=args["TempDir"]
sfOptions = {
"sfURL" : args['URL'],
"sfAccount" : args['ACCOUNT'],
"sfUser" : args['USERNAME'],
"sfPassword" : args['PASSWORD'],
"sfDatabase" : args['DB'],
"sfSchema" : args['SCHEMA'],
"sfRole" : args['ROLE'],
"sfWarehouse" : args['WAREHOUSE'],
"preactions" : "USE DATABASE dev_lz;",
}
#"tempDir" : tmp_dir,
print('=========DB Connection details ================== ', sfOptions)
datasource0 = glueContext.create_dynamic_frame.from_catalog(database = "aws-nonprod-datalake-glue-catalog", table_name = "nm_s_amaster", transformation_ctx = "datasource0")
applymapping1 = ApplyMapping.apply(frame = datasource0, mappings = [ mappings], transformation_ctx = "applymapping1")
selectfields2 = SelectFields.apply(frame = applymapping1, paths = [columns], transformation_ctx = "selectfields2")
resolvechoice3 = ResolveChoice.apply(frame = selectfields2, choice = "MATCH_CATALOG", database = "aws-nonprod-datalake-glue-catalog", table_name = "NM_TEMP", transformation_ctx = "resolvechoice3")
resolvechoice4 = ResolveChoice.apply(frame = resolvechoice3, choice = "make_cols", transformation_ctx = "resolvechoice4")
##Convert DataFrames to AWS Glue's DynamicFrames Object
resolvechoice4.toDF().write.format(SNOWFLAKE_SOURCE_NAME).options(**sfOptions).option("preactions","USE DATABASE dev_lz").option("dbtable", "nm_temp").mode("overwrite").save()
glueJob.commit()
But after running code i am getting
net.snowflake.client.jdbc.SnowflakeSQLException: SQL compilation error: Table 'NM_TEMP_STAGING_1100952600' does not exist
please let me know if I am missing anything.
I have permission for create, select stage, create, select table and create future tables.
above code I have removed columns and mappings. but original code it is available.
resolvechoice4.toDF().write.format(SNOWFLAKE_SOURCE_NAME).options(**sfOptions).option("preactions","USE DATABASE dev_lz").option("dbtable", "nm_temp").mode("overwrite").save()
Added following in above dbtable option it started working,
.option("preactions","USE ROLE DEVELOPER;USE DATABASE dev_db;USE SCHEMA aws_test")
as following
resolvechoice4.toDF().write.format(SNOWFLAKE_SOURCE_NAME).options(**sfOptions).option("preactions","USE DATABASE dev_lz").option("preactions","USE ROLE DEVELOPER;USE DATABASE dev_db;USE SCHEMA aws_test").option("dbtable", "nm_temp").mode("overwrite").save()
I want to save a new geometry data at database, but I'm receiving this error message all the time.
A .NET Framework error occurred during execution of user-defined routine or aggregate "geometry".
System.ArgumentException: 24100: The spatial reference identifier (SRID) is not valid. SRIDs must be between 0 and 999999.
Logging
2019-10-02 06:00:41.009 DEBUG 55688 --- [on(2)-127.0.0.1] o.h.e.j.e.i.JdbcEnvironmentInitiator : Database ->
name : Microsoft SQL Server
version : 14.00.1000
major : 14
minor : 0
2019-10-02 06:00:41.010 DEBUG 55688 --- [on(2)-127.0.0.1] o.h.e.j.e.i.JdbcEnvironmentInitiator : Driver ->
name : Microsoft JDBC Driver 7.4 for SQL Server
version : 7.4.1.0
major : 7
minor : 4
2019-10-02 06:00:41.010 DEBUG 55688 --- [on(2)-127.0.0.1] o.h.e.j.e.i.JdbcEnvironmentInitiator : JDBC version : 4.2
2019-10-02 06:00:41.054 INFO 55688 --- [on(2)-127.0.0.1] org.hibernate.dialect.Dialect : HHH000400: Using dialect: org.hibernate.spatial.dialect.sqlserver.SqlServer2008SpatialDialect
...
2019-10-02 05:50:22.232 DEBUG 62340 --- [nio-8080-exec-6] org.hibernate.SQL : insert into teste_geo (geom, nome) values (?, ?)
Hibernate: insert into teste_geo (geom, nome) values (?, ?)
2019-10-02 05:50:22.232 TRACE 62340 --- [nio-8080-exec-6] o.h.type.descriptor.sql.BasicBinder : binding parameter [1] as [VARBINARY] - [POLYGON ((4 0, 2 2, 4 4, 6 2, 4 0))]
2019-10-02 05:50:22.232 TRACE 62340 --- [nio-8080-exec-6] o.h.type.descriptor.sql.BasicBinder : binding parameter [2] as [VARCHAR] - [5f230d1b-ad0d-44a8-997e-02f4533bcfcd]
2019-10-02 05:50:26.452 INFO 62340 --- [ scheduling-1] c.v.g.o.service.ExemploService : Executou chamada do servico!
2019-10-02 05:50:26.452 WARN 62340 --- [nio-8080-exec-6] o.h.engine.jdbc.spi.SqlExceptionHelper : SQL Error: 6522, SQLState: S0001
My class code
import lombok.Getter;
import lombok.Setter;
import org.locationtech.jts.geom.Geometry;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
#Entity
#Table(name = "TESTE_GEO")
#Getter
#Setter
public class TesteGeom {
#Id
#Column(name = "nome")
private String name;
#Column(name = "geom")
private Geometry geometry;
}
...........
UUID idUnique = UUID.randomUUID();
TesteGeom t = new TesteGeom();
t.setName(idUnique.toString());
GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel() ,4326);
Coordinate[] coords =
new Coordinate[] {new Coordinate(4, 0), new Coordinate(2, 2),
new Coordinate(4, 4), new Coordinate(6, 2), new Coordinate(4, 0) };
LinearRing ring = geometryFactory.createLinearRing( coords );
LinearRing holes[] = null; // use LinearRing[] to represent holes
int SRID = geometryFactory.getSRID();
Polygon polygon = geometryFactory.createPolygon(ring, holes );
t.setGeometry(polygon);
t.getGeometry().setSRID(4326);
Executing the same SQL on management studio it works!
insert into teste_geo (geom, nome) values ('POLYGON ((4 0, 2 2, 4 4, 6 2, 4 0))', 'OK');
nome varchar(50)
geom geometry
In a query I got the error. Probably there are some dialect error.
org.springframework.orm.jpa.JpaSystemException: could not deserialize; nested exception is org.hibernate.type.SerializationException: could not deserialize
at org.springframework.orm.jpa.vendor.HibernateJpaDialect.convertHibernateAccessException(HibernateJpaDialect.java:351)
at org.springframework.orm.jpa.vendor.HibernateJpaDialect.translateExceptionIfPossible(HibernateJpaDialect.java:253)
at org.springframework.orm.jpa.AbstractEntityManagerFactoryBean.translateExceptionIfPossible(AbstractEntityManagerFactoryBean.java:527)
at org.springframework.dao.support.ChainedPersistenceExceptionTranslator.translateExceptionIfPossible(ChainedPersistenceExceptionTranslator.java:61)
I found the solution.
At my model I was using:
import org.locationtech.jts.geom.Geometry;
But Hibernate Spatial only accept :
import com.vividsolutions.jts.geom.Geometry;
So, I change my lib and everything work.
class PostTemplate(BaseModel):
content = TextField(unique=True)
class VkGroup(BaseModel):
group_id = IntegerField(unique=True)
class PostTemplateVkGroup(BaseModel):
"""
http://charlesleifer.com/blog/a-tour-of-tagging-schemas-many-to-many-bitmaps-and-more/
"""
group = ForeignKeyField(VkGroup)
post_template = ForeignKeyField(PostTemplate)
def get_posted_templates_for_group(group_id: int) -> Iterable:
"""Get posted templates.
Args:
group_id (int): id группы
"""
queries = (PostTemplate
.select()
.join(PostTemplateVkGroups)
.join(VkGroup)
.where(VkGroup.group_id == group_id))
return queries
all_post_templates = PostTemplate.select()
Many-to-many relationship.
For every record in PostTemplateVkGroup post template from this record is used in group from this record.
all_post_templates = not_posted_templates | posted_templates
How I can get not_posted_templates?
If your database supports the "EXCEPT" operation, you can:
all_post_templates = PostTemplate.alias().select()
post_templates = get_posted_templates_for_group(...)
difference = all_post_templates - post_templates
Sqlite example:
class Post(Base):
title = TextField()
class Tag(Base):
tag = TextField()
class PostTag(Base):
post = ForeignKeyField(Post)
tag = ForeignKeyField(Tag)
db.create_tables([Post, Tag, PostTag])
data = (
('pa', ('ta1', 'ta2')),
('pb', ('tb1', 'tb2')),
('pc', ()))
for title, tags in data:
post = Post.create(title=title)
for tag in tags:
tag = Tag.create(tag=tag)
PostTag.create(post=post, tag=tag)
# Create some tags that aren't associated with any post.
Tag.create(tag='tx1')
Tag.create(tag='tx2')
pa1_tags = (Tag
.select()
.join(PostTag)
.join(Post)
.where(Post.title == 'pa'))
all_tags = Tag.alias().select()
diff = all_tags - pa1_tags
for t in diff:
print(t.tag)
# Prints
# tb1
# tb2
# tx1
# tx2
I have created a basic setup for autocomplete-light on one of my forms. I am using direct SQL queries to access the data I require, and the Select2ListView to send the data to the template. The autocomplete data is available on the browser, but it is case sensitive. Can someone please suggest what I can do to make the autocomplete-light become case insensitive?
I read about using 'split_words = True' from the AutocompleteModel class, but I have no idea how to do this.
For example (q = 'ger'):
Command Prompt (views.py print statement) returns
['Algeria', 'Germany', 'Niger (the), 'Nigeria']
http://127.0.0.1:8000/autocomplete/country-autocomplete?q=ger returns
{"results": [{"text": "Algeria", "id": "Algeria"}, {"text": "Niger (the)", "id": "Niger (the)"}, {"text": "Nigeria", "id": "Nigeria"}]}
As you can see, Germany is visible in the data I am returning from the view, but there is some additional filtering going on by autocomplete-light. I need to change autocomplete-light so that the filtering it performs is case-insensitive. Any ideas?
I am running Python 3.5.2, and the latest pip install for autocomplete-light.
Thanks
Stephen
View
class CountryAutocomplete(autocomplete.Select2ListView):
def get_list(self):
# Only allow authenticated users
if not self.request.user.is_authenticated():
return []
# Read all country names from database
cursor = connection.cursor()
cursor.execute('SELECT country_name FROM autocomplete_iso3166;')
results = cursor.fetchall()
# Flatten database query into a list
results = [e for l in results for e in l]
# Check POST query and filter further
if self.q:
# Make query case insensitive
results = list(filter(lambda k: self.q.casefold() in str(k).casefold(), results))
print(results)
return results
Form
class VendorForm(forms.ModelForm):
class Meta:
model = client
fields = [
'vendor_name', 'address5', 'address4', 'address3', 'address2', 'address1',
]
widgets = {
'address5': autocomplete.ListSelect2(url='country-autocomplete'),
'address4': autocomplete.ListSelect2(url='state-autocomplete',forward=['address5'])
}
Model
class client(models.Model):
vendor_name = models.CharField(max_length=100)
user_admin = models.ForeignKey(User, unique=False, null=True, blank=True)
address1 = models.CharField(null=True, blank=True, max_length=200)
address2 = models.CharField(null=True, blank=True, max_length=50)
address3 = models.CharField(null=True, blank=True, max_length=50)
address4 = models.CharField(null=True, blank=True, max_length=50)
address5 = models.CharField(null=True, blank=True, max_length=50)
if self.q:
queryset = queryset.filter(country_name__istartswith=self.q)
i means case insensitive. This example uses the beginning of the field.
If you want an exact match case-insensitive it would be:
country_name__iexact=self.q
I got following problem in sqlalchemy. I made three different tables in sqlite, the first has no realtion, the second has a relation to the first and the third a relation to the second. So when I want to insert things in the first and the second table everything works fine. When I want to insert datas in the third table I'm getting troubles when I'm going to do it like it's discribed in the tutorial. Here is my code for the three tables:
First table:
# Save_Data_Type.py
from sqlalchemy import ForeignKey
from sqlalchemy import Column, Float, Integer, String
from base import Base
from sqlalchemy.orm import relationship, backref
########################################################################
class Save_Data_Type(Base):
__tablename__ = "save_datas_type"
save_datas_type_id = Column(Integer, primary_key=True)
type_memory = Column(String)
comment = Column(String)
dummy1 = Column(String)
dummy2 = Column(String)
#----------------------------------------------------------------------
def __init__(self, type_memory, comment, dummy1, dummy2):
""""""
self.type_memory = type_memory
self.comment = comment
self.dummy1 = dummy1
self.dummy2 = dummy2
Second Table
# Save_Data.py
from sqlalchemy import ForeignKey
from sqlalchemy import Column, Float, Integer, String
from base import Base
from sqlalchemy.orm import relationship, backref
########################################################################
class Save_Data(Base):
""""""
__tablename__ = "save_datas"
save_datas_id = Column(Integer, primary_key=True)
save_datas_type_id = Column(Integer,ForeignKey("save_datas_type.save_datas_type_id"))
save_datas_type = relationship("Save_Data_Type", backref=backref("save_datas", order_by=save_datas_id))
value = Column(Float)
comment = Column(String)
dummy1 = Column(String)
#----------------------------------------------------------------------
def __init__(self, value, comment, dummy1):
""""""
self.value = value
self.comment = comment
self.dummy1 = dummy1
Third Table
# Station.py
from sqlalchemy import ForeignKey
from sqlalchemy import Column, Integer, Boolean, String
from base import Base
from sqlalchemy.orm import relationship, backref
########################################################################
class Station(Base):
""""""
__tablename__ = "stations"
stations_id = Column(Integer, primary_key=True)
name = Column(String)
password = Column(String)
save_datas_id = Column(Integer,ForeignKey("save_datas.save_datas_id"))
save_datas = relationship("Save_Data", backref=backref("stations", order_by=stations_id))
dummy1 = Column(String)
dummy2 = Column(String)
dummy3 = Column(String)
#----------------------------------------------------------------------
def __init__(self, name, password, dummy1, dummy2, dummy3):
""""""
self.name = name
self.password = password
self.dummy1 = dummy1
self.dummy2 = dummy2
self.dummy3 = dummy3
base.py
# base.py
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
So if I'm going to insert the datas like that:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import base
from Save_Data_Type import Save_Data_Type
from Save_Data import Save_Data
from Station import Station
engine = create_engine('sqlite:///Database.db', echo=True)
base.Base.metadata.create_all(engine, checkfirst=True)
# create a Session
Session = sessionmaker(bind=engine)
session = Session()
jack = Save_Data_Type("Ring Memory",'In seconds',None,None)
jack.save_datas = [Save_Data(1980,'Sometimes more, sometimes less',None)]
jack.save_datas.stations = [Station('name1','123456',None,None,None)]
session.add(jack)
session.commit()
Nothing is writing in the the third table of the database. How is the usual way to build this relationship?
Thanks in advance,
Johannes
Apart from some typos in the code you posted (see comment from zzzeek), it looks like the problem will be solved by replacing the code:
jack.save_datas.stations = [Station('name1','123456',None,None,None)]
with the one below:
jack.save_datas[0].stations = [Station('name1','123456',None,None,None)]
The reason is the following: in your case you assign/create a stations attribute to a class member. Not only this does not do the right job, it might actualy harm your ORM model.
In the second case you correctly assign the Station instance to the first save_datas.
The more clear code would look like this:
jack = Save_Data_Type("Ring Memory",'In seconds',None,None)
save_data1 = Save_Data(1980,'Sometimes more, sometimes less',None)
jack.save_datas.append(save_data1)
# or: jack.save_datas = [save_data1]
# or (my favourite): save_data1.Save_Data_Type = jack
save_data1.stations = [Station('name1','123456',None,None,None)]
# or... similar to the relationship above