How to add new meta data to alembic file in maya? - maya

I want to add some extra information into the exists abc file or if its possible while creating alembic cache with some extra information in maya or any cg application using pyhon.
I am appreciate any one can help me to edit the alembic file.
input example
meta_data = {'name': 'Hero', 'tag': 'test_show'}
abc_file = '/show/test_show/scene/hero.abc'
set meta data ?
from alembic import Abc
get meta data
from alembic import Abc
archive = Abc.IArchive(abc_file)
top = archive.getTop()
meta_data = top.getMetaData()
print meta_data__str()

Here's a complete script that does the job of copying the source alembic file and inserting some metadata:
import os
import alembic
def copy_props(i_props, o_props):
'''
Copy properties
'''
for index in range(i_props.getNumProperties()):
header = i_props.getPropertyHeader(index)
if header.isArray():
i_prop = alembic.Abc.IArrayProperty(
i_props,
header.getName())
prop_name = i_prop.getName()
prop_meta = i_prop.getMetaData()
o_prop = alembic.Abc.OArrayProperty(
o_props,
prop_name,
i_prop.getDataType(),
prop_meta,
0)
o_prop.setTimeSampling(i_prop.getTimeSampling())
for i in range(i_prop.getNumSamples()):
o_prop.setValue(i_prop.getValue(i))
elif header.isScalar():
i_prop = alembic.Abc.IScalarProperty(
i_props,
header.getName())
prop_name = i_prop.getName()
prop_meta = i_prop.getMetaData()
o_prop = alembic.Abc.OScalarProperty(
o_props,
prop_name,
i_prop.getDataType(),
prop_meta,
0)
o_prop.setTimeSampling(i_prop.getTimeSampling())
for i in range(i_prop.getNumSamples()):
o_prop.setValue(i_prop.getValue(i))
elif header.isCompound():
i_prop = alembic.Abc.ICompoundProperty(
i_props,
header.getName())
prop_name = i_prop.getName()
prop_meta = i_prop.getMetaData()
o_prop = alembic.Abc.OCompoundProperty(
o_props,
prop_name,
prop_meta)
copy_props(i_prop, o_prop)
def copy_object(i_obj, o_obj):
'''
Recursively copy object data
'''
if o_obj is None:
return
i_props = i_obj.getProperties()
o_props = o_obj.getProperties()
copy_props(i_props, o_props)
for index in range(i_obj.getNumChildren()):
i_child = i_obj.getChild(index)
i_child_name = i_child.getName()
i_child_meta = i_child.getMetaData()
o_child = alembic.Abc.OObject(o_obj, i_child_name, i_child_meta)
copy_object(i_child, o_child)
def copy_abc(i_path, o_path, app, description):
'''
Copy alembic file from i_path to o_path
'''
arc_in = alembic.Abc.IArchive(i_path)
arc_out = alembic.Abc.OArchive(o_path, asOgawa=True)
arc_out = alembic.Abc.CreateArchiveWithInfo(o_path, app, description)
top_in = arc_in.getTop()
top_out = arc_out.getTop()
copy_object(top_in, top_out)
def read(abc_file):
archive = alembic.Abc.IArchive(abc_file)
return alembic.Abc.GetArchiveInfo(archive)
if __name__ == '__main__':
i_path = os.path.join(
os.path.dirname(__file__),
'itest.abc'
)
o_path = os.path.join(
os.path.dirname(__file__),
'otest.abc'
)
copy_abc(i_path, o_path, 'Cool app', 'Cool description')
print('Created archive: ' + o_path)
archive_info = read(o_path)
print('App name: ' + archive_info.get('appName'))
print('Description: ' + archive_info.get('userDescription'))
print('Written: ' + archive_info.get('whenWritten'))

You can't write just arbitrary data but you can set description and application strings:
from alembic import Abc
MY_APP = 'My cool application'
def write(abc_file, description):
archive = Abc.CreateArchiveWithInfo(abc_file, MY_APP, description)
def read(abc_file):
archive = Abc.IArchive(abc_file)
top = archive.getTop()
return Abc.GetArchiveInfo(archive)
abc_file = 'alembic.abc'
write(abc_file, 'An abc file cool description')
archive_info = read(abc_file)
print(archive_info.get('appName'))
print(archive_info.get('userDescription'))
print(archive_info.get('whenWritten'))

Related

PyFlink 14.2 - Table API DDL - Semantic Exactly Once

I've had the scenario where I define a kafka source, UDF | UDTF for processing and sink to a Kafka sink. Doesn't matter what I do, if I run the job, the output is flood with the processed output of a single input record. For illustrative purposes, this is what's output on the defined kafka sink topic:
distinct timestamps, showing that the UDF is entered for each respective input record, but the same input record was processed:
By trying to figure out the problem I've read through whatever flink documentation I could find (and rabbit hole of links) in terms of enforcing 'semantic EXACTLY ONCE' processing of records. As far as I can gather it comes down to these following settings:
This guy presented the best visual representation for me to understand semantic_once_video
Kafka source (consumer)
kafka source property of isolation level = read_committed
Kafka sinks (producer)
Kafka sink property of processing mode = exactly_once
Kafka sink property of idempotence = true
Utilizing checkpointing
Also referencing stackoverflow questions I could find on the topic (mainly discussing in terms of Java implementations)... needless to say, still not resolved. Here's my code for reference:
import os
from pyflink.datastream.stream_execution_environment import StreamExecutionEnvironment
from pyflink.table import TableEnvironment, EnvironmentSettings, DataTypes, StreamTableEnvironment
from pyflink.table.udf import ScalarFunction, TableFunction, udf, udtf
from pyflink.datastream.checkpointing_mode import CheckpointingMode
KAFKA_SERVERS = os.getenv('KAFKA_BS_SERVERS',"localhost:9094").split(',')
KAFKA_USERNAME = "xxx"
KAFKA_PASSWORD = "_pass_"
KAFKA_SOURCE_TOPIC = 'source_topic'
KAFKA_SINK_TOPIC = 'sink_topic'
KAFKA_GROUP_ID = 'testgroup12'
JAR_DEPENDENCIES = os.getenv('JAR_DEPENDENCIES', '/opt/flink/lib_py')
class tbl_function(TableFunction):
def open(self, function_context):
pass
def eval(self, *args):
import json
from datetime import datetime
res = {
'time': str(datetime.utcnow()),
'input': json.loads(args[0])
}
yield json.dumps(res)
def pipeline():
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
for file in os.listdir(JAR_DEPENDENCIES):
if file.find('.jar') != -1:
env.add_jars(f"file://{JAR_DEPENDENCIES}/{file}")
print(f"added jar dep: {JAR_DEPENDENCIES}/{file}")
env.enable_checkpointing(60000, CheckpointingMode.EXACTLY_ONCE)
env.get_checkpoint_config().set_min_pause_between_checkpoints(120000)
env.get_checkpoint_config().enable_unaligned_checkpoints()
env.get_checkpoint_config().set_checkpoint_interval(30000)
settings = EnvironmentSettings.new_instance()\
.in_streaming_mode()\
.use_blink_planner()\
.build()
t_env = StreamTableEnvironment.create(stream_execution_environment= env, environment_settings=settings)
source_ddl = f"""
CREATE TABLE source_table(
entry STRING
) WITH (
'connector' = 'kafka',
'topic' = '{KAFKA_SOURCE_TOPIC}',
'properties.bootstrap.servers' = '{','.join(KAFKA_SERVERS)}',
'properties.isolation_level' = 'read_committed',
'properties.group.id' = '{KAFKA_GROUP_ID}',
'properties.sasl.mechanism' = 'PLAIN',
'properties.security.protocol' = 'SASL_PLAINTEXT',
'properties.sasl.jaas.config' = 'org.apache.kafka.common.security.plain.PlainLoginModule required username=\"{KAFKA_USERNAME}\" password=\"{KAFKA_PASSWORD}\";',
'scan.startup.mode' = 'earliest-offset',
'format' = 'raw'
)
"""
sink_ddl = f"""
CREATE TABLE sink_table(
entry STRING
) WITH (
'connector' = 'kafka',
'topic' = '{KAFKA_SINK_TOPIC}',
'properties.bootstrap.servers' = '{','.join(KAFKA_SERVERS)}',
'properties.group.id' = '{KAFKA_GROUP_ID}',
'properties.processing.mode' = 'exactly_once',
'properties.enable.idempotence' = 'true',
'properties.sasl.mechanism' = 'PLAIN',
'properties.security.protocol' = 'SASL_PLAINTEXT',
'properties.sasl.jaas.config' = 'org.apache.kafka.common.security.plain.PlainLoginModule required username=\"{KAFKA_USERNAME}\" password=\"{KAFKA_PASSWORD}\";',
'format' = 'raw'
)
"""
t_env.execute_sql(source_ddl).wait()
t_env.execute_sql(sink_ddl).wait()
f = tbl_function()
table_udf = udtf(f, result_types=[DataTypes.STRING()])
t_env.create_temporary_function("table_f", table_udf)
table = t_env.from_path('source_table')
table = table.join_lateral('table_f(entry) as (content)')
table = table.select('content').alias('entry')
table.insert_into('sink_table')
from datetime import datetime
t_env.execute(f"dummy_test_{str(datetime.now())}")
if __name__ == '__main__':
pipeline()
jar dependencies:
added jar dep: /opt/flink/lib_py/flink-sql-connector-kafka_2.12-1.14.2.jar
added jar dep: /opt/flink/lib_py/flink-connector-kafka_2.12-1.14.2.jar
added jar dep: /opt/flink/lib_py/kafka-clients-2.4.1.jar
After a whole bunch of trial-error, and still not precisely knowing why this resolved the issue (or underlying pyflink issue?), I found that if you utilize a table meta-data field in your source definition, that would somehow initialize or synchronize your pipeline to produce a semantic.EXACTLY_ONCE data flow (1 record in = 1 record out, no duplicates).
The only change that I had to made is 1 line of meta data code in the DDL source definition. (Again providing my full script for reference):
import os
from pyflink.datastream.stream_execution_environment import StreamExecutionEnvironment
from pyflink.table import TableEnvironment, EnvironmentSettings, DataTypes, StreamTableEnvironment
from pyflink.table.udf import ScalarFunction, TableFunction, udf, udtf
from pyflink.datastream.checkpointing_mode import CheckpointingMode
KAFKA_SERVERS = os.getenv('KAFKA_BS_SERVERS',"localhost:9094").split(',')
KAFKA_USERNAME = "xxx"
KAFKA_PASSWORD = "_pass_"
KAFKA_SOURCE_TOPIC = 'source_topic'
KAFKA_SINK_TOPIC = 'sink_topic'
KAFKA_GROUP_ID = 'testgroup12'
JAR_DEPENDENCIES = os.getenv('JAR_DEPENDENCIES', '/opt/flink/lib_py')
class tbl_function(TableFunction):
def open(self, function_context):
pass
def eval(self, *args):
import json
from datetime import datetime
res = {
'time': str(datetime.utcnow()),
'input': json.loads(args[0])
}
yield json.dumps(res)
def pipeline():
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
for file in os.listdir(JAR_DEPENDENCIES):
if file.find('.jar') != -1:
env.add_jars(f"file://{JAR_DEPENDENCIES}/{file}")
print(f"added jar dep: {JAR_DEPENDENCIES}/{file}")
env.enable_checkpointing(60000, CheckpointingMode.EXACTLY_ONCE)
env.get_checkpoint_config().set_min_pause_between_checkpoints(120000)
env.get_checkpoint_config().enable_unaligned_checkpoints()
env.get_checkpoint_config().set_checkpoint_interval(30000)
settings = EnvironmentSettings.new_instance()\
.in_streaming_mode()\
.use_blink_planner()\
.build()
t_env = StreamTableEnvironment.create(stream_execution_environment= env, environment_settings=settings)
# this sneaky bugger line -> with 'event_time'
source_ddl = f"""
CREATE TABLE source_table(
entry STRING,
event_time TIMESTAMP(3) METADATA FROM 'timestamp'
) WITH (
'connector' = 'kafka',
'topic' = '{KAFKA_SOURCE_TOPIC}',
'properties.bootstrap.servers' = '{','.join(KAFKA_SERVERS)}',
'properties.isolation_level' = 'read_committed',
'properties.group.id' = '{KAFKA_GROUP_ID}',
'properties.sasl.mechanism' = 'PLAIN',
'properties.security.protocol' = 'SASL_PLAINTEXT',
'properties.sasl.jaas.config' = 'org.apache.kafka.common.security.plain.PlainLoginModule required username=\"{KAFKA_USERNAME}\" password=\"{KAFKA_PASSWORD}\";',
'scan.startup.mode' = 'earliest-offset',
'format' = 'raw'
)
"""
sink_ddl = f"""
CREATE TABLE sink_table(
entry STRING
) WITH (
'connector' = 'kafka',
'topic' = '{KAFKA_SINK_TOPIC}',
'properties.bootstrap.servers' = '{','.join(KAFKA_SERVERS)}',
'properties.group.id' = '{KAFKA_GROUP_ID}',
'properties.processing.mode' = 'exactly_once',
'properties.enable.idempotence' = 'true',
'properties.sasl.mechanism' = 'PLAIN',
'properties.security.protocol' = 'SASL_PLAINTEXT',
'properties.sasl.jaas.config' = 'org.apache.kafka.common.security.plain.PlainLoginModule required username=\"{KAFKA_USERNAME}\" password=\"{KAFKA_PASSWORD}\";',
'format' = 'raw'
)
"""
t_env.execute_sql(source_ddl).wait()
t_env.execute_sql(sink_ddl).wait()
f = tbl_function()
table_udf = udtf(f, result_types=[DataTypes.STRING()])
t_env.create_temporary_function("table_f", table_udf)
table = t_env.from_path('source_table')
table = table.join_lateral('table_f(entry) as (content)')
table = table.select('content').alias('entry')
table.insert_into('sink_table')
from datetime import datetime
t_env.execute(f"dummy_test_{str(datetime.now())}")
if __name__ == '__main__':
pipeline()
Hope this saves someone time, unlike the 3 days I spent in trial-error #Sigh :(

SQLAlchemy and QThread: Session handling in multi-thread

I have a problem: I don't know how to work with session, in combination with multi-thread, efficiently. Well you can see that I use scoped_session. It will transparently creates a thread-local session - so its threadsafe.
My below and executable example works, as long as the interval of QTimeris 1000, but if you set the value to 1, then there are some problems. On GUI-site, there a 8 QComboBox()-objects, and I will start 8 threads. In this example, I work with one table. When I run this program, not all QComboBox()-objects are filled. Sometimes there is only one QComboBox()-object blank or more. In addition, it happens once in a while that I am told by SQLAlchemy that the connection was closed. For this I have an error message:
Traceback (most recent call last):
File "D:\Dan\Python\Xarphus\xarphus\subclass_master_data_load_data_item.py", line 151, in populate_item
self.populate_item_signal.emit(next(self._element))
File "D:\Dan\Python\Xarphus\xarphus\core\manage_data_manipulation_master_data.py", line 232, in select_all
yield record.id, record.relationship
File "D:\Dan\Python\Xarphus\xarphus\core\manage_db_connection.py", line 245, in __exit__
self.session.commit()
File "C:\Python27\lib\site-packages\sqlalchemy\orm\session.py", line 906, in commit
self.transaction.commit()
File "C:\Python27\lib\site-packages\sqlalchemy\orm\session.py", line 465, in commit
t[1].commit()
File "C:\Python27\lib\site-packages\sqlalchemy\engine\base.py", line 1632, in commit
self._do_commit()
File "C:\Python27\lib\site-packages\sqlalchemy\engine\base.py", line 1663, in _do_commit
self.connection._commit_impl()
File "C:\Python27\lib\site-packages\sqlalchemy\engine\base.py", line 726, in _commit_impl
self.connection._reset_agent is self.__transaction:
File "C:\Python27\lib\site-packages\sqlalchemy\engine\base.py", line 351, in connection
self._handle_dbapi_exception(e, None, None, None, None)
File "C:\Python27\lib\site-packages\sqlalchemy\engine\base.py", line 1405, in _handle_dbapi_exception
util.reraise(*exc_info)
File "C:\Python27\lib\site-packages\sqlalchemy\engine\base.py", line 349, in connection
return self._revalidate_connection()
File "C:\Python27\lib\site-packages\sqlalchemy\engine\base.py", line 429, in _revalidate_connection
raise exc.ResourceClosedError("This Connection is closed")
ResourceClosedError: This Connection is closed
And, if I run my program several times, I get, among other things sometimes the following message:
Traceback (most recent call last):
File "C:\Users\Sophus\Desktop\ver_2_simple_problem.py", line 83, in init_object
self._element = self.master_data_manipulation.select_all()
File "C:\Users\Sophus\Desktop\ver_2_simple_problem.py", line 178, in __exit__
self.session.commit()
File "C:\Python27\lib\site-packages\sqlalchemy\orm\session.py", line 906, in commit
self.transaction.commit()
File "C:\Python27\lib\site-packages\sqlalchemy\orm\session.py", line 459, in commit
self._assert_active(prepared_ok=True)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\session.py", line 258, in _assert_active
"This session is in 'committed' state; no further "
sqlalchemy.exc.InvalidRequestError: This session is in 'committed' state; no further SQL can be emitted within this transaction.
Here is my example code: Is there a better and more elegant source code on how to work with SQLAlchemy in multiple threads? Becaouse I'm not sure if this is the right way.
from PyQt4.QtCore import QObject, QThread, pyqtSignal, pyqtSlot, QTimer
from PyQt4.QtGui import QApplication, QPushButton, QVBoxLayout, QDialog, \
QComboBox, QLabel
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import sessionmaker, scoped_session
from sqlalchemy import Table, Column, Integer, String, MetaData
from traceback import format_exc
from sys import exc_info
''' setting up root class for declarative declaration '''
Base = declarative_base()
class PERSON_SALUTATION(Base):
__tablename__ = "person_salutation"
id = Column(Integer, primary_key=True)
salutation = Column(String(50), nullable=False, unique=True)
class MasterDataManipulation(object):
def __init__(self, session_object=None):
self._session_scope = session_object
def select_all(self):
try:
with self._session_scope as session:
for record in session.query(PERSON_SALUTATION):
yield record.id, record.salutation
except AttributeError:
print "select all, desired_trace", format_exc(exc_info())
return
class Worker(QObject):
finish_progress = pyqtSignal()
populate_item_signal = pyqtSignal(object, object)
def __init__(self,
combo_box=None,
query_data=None,
parent=None):
QObject.__init__(self, parent)
self.query_data = query_data
self.combo_box=combo_box
''' Create attributes '''
self._run_semaphore = 1
def init_object(self):
self._element = self.query_data()
self.timer = QTimer()
self.timer.setSingleShot(False)
self.timer.setInterval(1)
self.timer.timeout.connect(self.populate_item)
self.timer.start()
def populate_item(self):
try:
if self._run_semaphore == 0:
self._run_semaphore = 1
raise StopIteration
else:
self.populate_item_signal.emit(next(self._element), self.combo_box)
except StopIteration:
print "StopIteration is raised"
self.timer.stop()
def stop(self):
self._run_semaphore == 0
self.timer.stop()
class SessionScope(object):
def __init__(self, dbms=None, dbdriver=None,
dbuser=None, dbuser_pwd=None,
db_server_host=None, dbport=None, db_name=None,
admin_database=None):
self.dbms = dbms
self.dbdriver = dbdriver
self.dbuser = dbuser
self.dbuser_pwd = dbuser_pwd
self.db_server_host = db_server_host
self.dbport = dbport
self.db_name = db_name
self.admin_database = admin_database
url = '{}+{}://{}:{}#{}:{}/{}'.format(
self.dbms, self.dbdriver, self.dbuser, self.dbuser_pwd, self.db_server_host, self.dbport, self.db_name)
self._Engine = create_engine(url, encoding='utf8', echo=True)
self.session = None
self._session_factory = sessionmaker(bind=self._Engine)
self._Session = scoped_session(sessionmaker(bind=self._Engine, expire_on_commit=False))
''' create tables '''
Base.metadata.create_all(self._Engine)
def __enter__(self):
self.session = self._Session()
return self.session
def __exit__(self, exception, exc_value, traceback):
try:
if exception:
self.session.rollback()
else:
self.session.commit()
finally:
self.session.close()
class MyCustomDialog(QDialog):
finish = pyqtSignal()
def __init__(self, scoped_session=None, parent=None):
QDialog.__init__(self, parent)
self._session_scope = scoped_session
self._list_threads = []
self.init_ui()
self.start_all_selection()
def init_ui(self):
layout = QVBoxLayout(self)
self.combo_person_title = QComboBox(self)
self.combo_person_salutation = QComboBox(self)
self.combo_person_gender = QComboBox(self)
self.combo_person_religion = QComboBox(self)
self.combo_person_relationship_status = QComboBox(self)
self.combo_person_nationality = QComboBox(self)
self.combo_person_eye_color = QComboBox(self)
self.combo_person_hair_color = QComboBox(self)
self.pushButton_populate_combo = QPushButton("Re-populate", self)
self.pushButton_stopp = QPushButton("Stopp", self)
self.pushButton_close = QPushButton("Close", self)
layout.addWidget(self.combo_person_title)
layout.addWidget(self.combo_person_salutation)
layout.addWidget(self.combo_person_gender)
layout.addWidget(self.combo_person_religion)
layout.addWidget(self.combo_person_nationality)
layout.addWidget(self.combo_person_relationship_status)
layout.addWidget(self.combo_person_eye_color)
layout.addWidget(self.combo_person_hair_color)
layout.addWidget(self.pushButton_populate_combo)
layout.addWidget(self.pushButton_stopp)
layout.addWidget(self.pushButton_close)
self.pushButton_stopp.clicked.connect(self.on_finish)
self.pushButton_populate_combo.clicked.connect(self.start_all_selection)
self.pushButton_close.clicked.connect(self.close)
def start_all_selection(self):
list_comboxes = self.findChildren(QComboBox)
for combo_box in list_comboxes:
combo_box.clear()
self.start_thread(combo_box=combo_box)
def fill_combo_boxt(self, item, combo_box):
id, text = item
combo_box.addItem(text)
def on_label(self, i):
self.label.setText("Result: {}".format(i))
def start_thread(self, combo_box=None):
master_data_manipulation = MasterDataManipulation(session_object=self._session_scope)
query_data=master_data_manipulation.select_all
task_thread = QThread(self)
task_thread.work = Worker(query_data=query_data,
combo_box=combo_box,)
''' We need to store threads '''
self._list_threads.append(task_thread)
task_thread.work.moveToThread(task_thread)
task_thread.work.populate_item_signal.connect(self.fill_combo_boxt)
self.finish.connect(task_thread.work.stop)
task_thread.started.connect(task_thread.work.init_object)
task_thread.finished.connect(task_thread.deleteLater)
''' This will emit 'started' and start thread's event loop '''
task_thread.start()
#pyqtSlot()
def abort_workers(self):
self.finish.emit()
for thread in self._list_threads:
''' this will quit **as soon as thread event loop unblocks** '''
thread.quit()
''' so you need to wait for it to *actually* quit'''
thread.wait()
def on_finish(self):
self.finish.emit()
def closeEvent(self, event):
''' Re-implementaate to handle with created threads '''
self.abort_workers()
sys.exit()
def populate_database(sess=None):
try:
with sess as session:
salutations = [PERSON_SALUTATION(salutation="Mister"),
PERSON_SALUTATION(salutation="Miss"),
PERSON_SALUTATION(salutation="Lady"),
PERSON_SALUTATION(salutation="Ma'am"),
PERSON_SALUTATION(salutation="Sir"),
PERSON_SALUTATION(salutation="Queen"),
PERSON_SALUTATION(salutation="Grandma"),]
session.add_all(salutations)
session.commit()
except SQLAlchemyError:
print "SQLAlchemyError", format_exc(exc_info())
def main():
dbms = raw_input('Enter database type: ')
dbdriver = raw_input('Enter database driver: ')
dbuser = raw_input('Enter user name: ')
dbuser_pwd = raw_input('Enter user password: ')
db_server_host = raw_input('Enter server host: ')
dbport = raw_input('Enter port: ')
db_name = raw_input('Enter database name: ')
try:
''' create_engine and scoped_session once per process (per database). '''
session_scope = SessionScope(dbms = dbms,
dbdriver = dbdriver,
dbuser = dbuser,
dbuser_pwd = dbuser_pwd,
db_server_host = db_server_host,
dbport = dbport,
db_name = db_name)
answer = raw_input('Do you want to populate database? Type yes or no: ')
if answer.lower() == 'yes':
populate_database(sess=session_scope)
app = QApplication(sys.argv)
window = MyCustomDialog(scoped_session = session_scope)
window.show()
sys.exit(app.exec_())
except TypeError:
print "ERROR", format_exc(exc_info())
if __name__ == "__main__":
main()

Wipe out dropout operations from TensorFlow graph

I have a trained freezed graph that I am trying to run on an ARM device. Basically, I am using contrib/pi_examples/label_image, but with my network instead of Inception. My network was trained with dropout, which now causes me troubles:
Invalid argument: No OpKernel was registered to support Op 'Switch' with these attrs. Registered kernels:
device='CPU'; T in [DT_FLOAT]
device='CPU'; T in [DT_INT32]
device='GPU'; T in [DT_STRING]
device='GPU'; T in [DT_BOOL]
device='GPU'; T in [DT_INT32]
device='GPU'; T in [DT_FLOAT]
[[Node: l_fc1_dropout/cond/Switch = Switch[T=DT_BOOL](is_training_pl, is_training_pl)]]
One solution I can see is to build such TF static library that includes the corresponding operation. From other hand, it might be a better idea to eliminate the dropout ops from the network in order to make it simpler and faster. Is there a way to do that?
Thanks.
#!/usr/bin/env python2
import argparse
import tensorflow as tf
from google.protobuf import text_format
from tensorflow.core.framework import graph_pb2
from tensorflow.core.framework import node_def_pb2
def print_graph(input_graph):
for node in input_graph.node:
print "{0} : {1} ( {2} )".format(node.name, node.op, node.input)
def strip(input_graph, drop_scope, input_before, output_after, pl_name):
input_nodes = input_graph.node
nodes_after_strip = []
for node in input_nodes:
print "{0} : {1} ( {2} )".format(node.name, node.op, node.input)
if node.name.startswith(drop_scope + '/'):
continue
if node.name == pl_name:
continue
new_node = node_def_pb2.NodeDef()
new_node.CopyFrom(node)
if new_node.name == output_after:
new_input = []
for node_name in new_node.input:
if node_name == drop_scope + '/cond/Merge':
new_input.append(input_before)
else:
new_input.append(node_name)
del new_node.input[:]
new_node.input.extend(new_input)
nodes_after_strip.append(new_node)
output_graph = graph_pb2.GraphDef()
output_graph.node.extend(nodes_after_strip)
return output_graph
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--input-graph', action='store', dest='input_graph')
parser.add_argument('--input-binary', action='store_true', default=True, dest='input_binary')
parser.add_argument('--output-graph', action='store', dest='output_graph')
parser.add_argument('--output-binary', action='store_true', dest='output_binary', default=True)
args = parser.parse_args()
input_graph = args.input_graph
input_binary = args.input_binary
output_graph = args.output_graph
output_binary = args.output_binary
if not tf.gfile.Exists(input_graph):
print("Input graph file '" + input_graph + "' does not exist!")
return
input_graph_def = tf.GraphDef()
mode = "rb" if input_binary else "r"
with tf.gfile.FastGFile(input_graph, mode) as f:
if input_binary:
input_graph_def.ParseFromString(f.read())
else:
text_format.Merge(f.read().decode("utf-8"), input_graph_def)
print "Before:"
print_graph(input_graph_def)
output_graph_def = strip(input_graph_def, u'l_fc1_dropout', u'l_fc1/Relu', u'prediction/MatMul', u'is_training_pl')
print "After:"
print_graph(output_graph_def)
if output_binary:
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
else:
with tf.gfile.GFile(output_graph, "w") as f:
f.write(text_format.MessageToString(output_graph_def))
print("%d ops in the final graph." % len(output_graph_def.node))
if __name__ == "__main__":
main()
How about this as a more general solution:
for node in temp_graph_def.node:
for idx, i in enumerate(node.input):
input_clean = node_name_from_input(i)
if input_clean.endswith('/cond/Merge') and input_clean.split('/')[-3].startswith('dropout'):
identity = node_from_map(input_node_map, i).input[0]
assert identity.split('/')[-1] == 'Identity'
parent = node_from_map(input_node_map, node_from_map(input_node_map, identity).input[0])
pred_id = parent.input[1]
assert pred_id.split('/')[-1] == 'pred_id'
good = parent.input[0]
node.input[idx] = good

Waf:Create custom parallel tasks

In Waf how can I create multiple custom tasks, that can run parallel (with --jobs=JOBS)?
Sources = ["C:\\src1.c", "C:\\Mod1\src2.c", ... 30pcs] # one per call
Incl_Paths = ["Mod1". "Mod1"] # list all of them in all call
INCL_ST = "-I%s" # how to format an include path in an argument
Ext_out = "_loc" # output file extension
The goal:
C:\\LOC.exe -IMod1 -IMod2 C:\\src1.c > build\\src1.c_loc //or better src1_loc
C:\\LOC.exe -IMod1 -IMod2 C:\\Mod1\src2.c > build\\src2.c_loc //or better src2_loc
...
I couldn't get it work
def build(bld):
for i in Sources:
bld.new_task_gen(
source = i,
rule='C:\\LOC.exe ${INCL_ST:Incl_Paths} ${SRC} > ' + i + Ext_out,
)
Also I couldn't extract the exe
# find_program(self, filename, path_list=[], var=None, environ=None, exts=''):
cfg.find_program("C:\\LOC.exe", var='LOC')
To change from:
rule='C:\\LOC.exe ...'
To:
rule='${LOC} ...'
Something like this should work with waf 1.7:
from waflib.Task import Task
from waflib.TaskGen import extension
Ext_out = "_loc" # output file extension
def configure(conf):
# loc.exe must be in the system path for this to work
conf.find_program(
'loc',
var = "LOC",
)
conf.env.Incl_Paths = ["Mod1", "Mod1"]
conf.env.INCL_ST = "-I%s"
#extension('.c')
def process_loc(self, node):
out_node = node.change_ext(Ext_out)
tsk = self.create_task('loc')
tsk.set_inputs(node)
tsk.set_outputs(out_node)
class loc_task(Task):
ext_in = ['.c']
ext_out = ['_loc']
run_str = "${LOC} ${INCL_ST:Incl_Paths} ${SRC} > ${TGT}"
def build(bld):
bld(source = ["src1.c", "src2.c"])
Well it works for me on linux faking loc ...

'Model is not immutable' TypeError

I am getting this traceback;
--- Trimmed parts ---
File "C:\Users\muhammed\Desktop\gifdatabase\gifdatabase.py", line 76, in maketransaction
gif.tags = list(set(gif.tags + tags))
File "C:\Program Files (x86)\Google\google_appengine\google\appengine\ext\ndb\model.py", line 2893, in __hash__
raise TypeError('Model is not immutable')
TypeError: Model is not immutable
Here is related parts of my code;
class Gif(ndb.Model):
author = ndb.UserProperty()
#tags = ndb.StringProperty(repeated=True)
tags = ndb.KeyProperty(repeated=True)
#classmethod
def get_by_tag(cls,tag_name):
return cls.query(cls.tags == ndb.Key(Tag, tag_name)).fetch()
class Tag(ndb.Model):
gif_count = ndb.IntegerProperty()
class PostGif(webapp2.RequestHandler):
def post(self):
user = users.get_current_user()
if user is None:
self.redirect(users.create_login_url("/static/submit.html"))
return
link = self.request.get('gif_link')
tag_names = shlex.split(self.request.get('tags').lower())
#ndb.transactional(xg=True)
def maketransaction():
tags = [Tag.get_or_insert(tag_name) for tag_name in tag_names]
gif = Gif.get_or_insert(link)
if not gif.author: # first time submission
gif.author = user
gif.tags = list(set(gif.tags + tags))
gif.put()
for tag in tags:
tag.gif_count += 1
tag.put()
if validate_link(link) and tag_names:
maketransaction()
self.redirect('/static/submit_successful.html')
else:
self.redirect('/static/submit_fail.html')
What is the problem with gif.tags = list(set(gif.tags + tags)) line?
You are inserting tags instead of keys, you need to access
tags = [Tag.get_or_insert(tag_name).key .....]
but you can also make this a single network hop like this
futures = [Tag.get_or_insert_async(tag_name) for tag_name in tag_names]
futures.append(Gif.get_or_insert_async(link))
ndb.Future.wait_all(futures)
gif = futures.pop().get_result()
tags = [future.get_result() for future in futures]
but that's not really the question just a suggestion ^, for clearer answer with .key is
gif.tags = gif.tags + [tag.key for tag in tags]
# or
gif.tags.extend([tag.key for tag in tags])

Resources