Related
I modified Distiller (https://github.com/IntelLabs/distiller) to emulate in-memory computing circuit, especially added a convolution layer quantization during QAT. However, accuracy drops over 60% (90% > 30%) even with 32b quantization for sanity check. I also want to say that it was perfectly fine when I just add a calculated quantization noise. Below is the code.
import torch
import torch.nn as nn
import math
__all__ = ['preact_resnet20_cifar', 'preact_resnet32_cifar', 'preact_resnet44_cifar', 'preact_resnet56_cifar',
'preact_resnet110_cifar', 'preact_resnet20_cifar_conv_ds', 'preact_resnet32_cifar_conv_ds',
'preact_resnet44_cifar_conv_ds', 'preact_resnet56_cifar_conv_ds', 'preact_resnet110_cifar_conv_ds']
NUM_CLASSES = 10
device = torch.device("cuda")
def conv3x3(in_planes, out_planes, stride=1):
"""3x3 convolution with padding"""
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
def quantize(tensor, n_bits=32, dim=144, clip_ratio=1):
max_value = dim*clip_ratio
min_value = -dim*clip_ratio
delta = max_value - min_value
y = tensor.clone()
y = torch.clamp(y, min = min_value, max = max_value)
lsb = delta / (2**n_bits)
y = (y // lsb)*lsb
return y
class PreactBasicBlock(nn.Module):
expansion = 1
def __init__(self, block_gates, inplanes, planes, stride=1, downsample=None, preact_downsample=True, resolution = 32):
super(PreactBasicBlock, self).__init__()
self.block_gates = block_gates
self.pre_bn = nn.BatchNorm2d(inplanes)
self.pre_relu = nn.ReLU(inplace=False) # To enable layer removal inplace must be False
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn = nn.BatchNorm2d(planes)
self.relu = nn.ReLU(inplace=False)
self.conv2 = conv3x3(planes, planes)
self.downsample = downsample
self.stride = stride
self.preact_downsample = preact_downsample
self.resolution = resolution
def forward(self, x):
need_preact = self.block_gates[0] or self.block_gates[1] or self.downsample and self.preact_downsample # add pre-activation for block_gates 0 and 1 and when is downsampled
if need_preact: # x > bn > relu > out
preact = self.pre_bn(x)
preact = self.pre_relu(preact)
out = preact
else: # x > out
preact = out = x
if self.block_gates[0]: # (preact) > conv > bn > relu
out = self.conv1(out)
dim = self.conv1.in_channels * self.conv1.kernel_size[0] * self.conv1.kernel_size[1]
out = quantize(out, n_bits=self.resolution, dim=dim, clip_ratio=1)
out = self.bn(out)
out = self.relu(out)
if self.block_gates[1]: # (preact)> conv
out = self.conv2(out)
dim = self.conv2.in_channels * self.conv2.kernel_size[0] * self.conv2.kernel_size[1]
out = quantize(out, n_bits=self.resolution, dim=dim, clip_ratio=1)
if self.downsample is not None:
if self.preact_downsample:
residual = self.downsample(preact)
else:
residual = self.downsample(x)
else:
residual = x
out += residual
return out
class PreactResNetCifar(nn.Module):
def __init__(self, block, layers, num_classes=NUM_CLASSES, conv_downsample=False):
self.nlayers = 0
# Each layer manages its own gates
self.layer_gates = []
for layer in range(3):
# For each of the 3 layers, create block gates: each block has two layers
self.layer_gates.append([]) # [True, True] * layers[layer])
for blk in range(layers[layer]):
self.layer_gates[layer].append([True, True])
self.inplanes = 16 # 64
super(PreactResNetCifar, self).__init__()
self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
self.layer1 = self._make_layer(self.layer_gates[0], block, 16, layers[0],
conv_downsample=conv_downsample)
self.layer2 = self._make_layer(self.layer_gates[1], block, 32, layers[1], stride=2,
conv_downsample=conv_downsample)
self.layer3 = self._make_layer(self.layer_gates[2], block, 64, layers[2], stride=2,
conv_downsample=conv_downsample)
self.final_bn = nn.BatchNorm2d(64 * block.expansion)
self.final_relu = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(8, stride=1)
self.fc = nn.Linear(64 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def _make_layer(self, layer_gates, block, planes, blocks, stride=1, conv_downsample=False):
downsample = None
outplanes = planes * block.expansion
if stride != 1 or self.inplanes != outplanes:
if conv_downsample:
downsample = nn.Conv2d(self.inplanes, outplanes,
kernel_size=1, stride=stride, bias=False)
else:
# Identity downsample uses strided average pooling + padding instead of convolution
pad_amount = int(self.inplanes / 2)
downsample = nn.Sequential(
nn.AvgPool2d(2),
nn.ConstantPad3d((0, 0, 0, 0, pad_amount, pad_amount), 0)
)
layers = []
layers.append(block(layer_gates[0], self.inplanes, planes, stride, downsample, conv_downsample))
self.inplanes = outplanes
for i in range(1, blocks):
layers.append(block(layer_gates[i], self.inplanes, planes))
return nn.Sequential(*layers)
def forward(self, x):
x = self.conv1(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.final_bn(x)
x = self.final_relu(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
return x
def preact_resnet20_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [3, 3, 3], **kwargs)
return model
def preact_resnet32_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [5, 5, 5], **kwargs)
return model
def preact_resnet44_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [7, 7, 7], **kwargs)
return model
def preact_resnet56_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [9, 9, 9], **kwargs)
return model
def preact_resnet110_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [18, 18, 18], **kwargs)
return model
def preact_resnet182_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [30, 30, 30], **kwargs)
return model
def preact_resnet20_cifar_conv_ds(**kwargs):
return preact_resnet20_cifar(conv_downsample=True)
def preact_resnet32_cifar_conv_ds(**kwargs):
return preact_resnet32_cifar(conv_downsample=True)
def preact_resnet44_cifar_conv_ds(**kwargs):
return preact_resnet44_cifar(conv_downsample=True)
def preact_resnet56_cifar_conv_ds(**kwargs):
return preact_resnet56_cifar(conv_downsample=True)
def preact_resnet110_cifar_conv_ds(**kwargs):
return preact_resnet110_cifar(conv_downsample=True)
def preact_resnet182_cifar_conv_ds(**kwargs):
return preact_resnet182_cifar(conv_downsample=True)
I use distiller.example.classifier_compression.compress_classifier.py in Distiller, and use terminal with schedule file. Command is "python compress_classifier.py -a preact_resnet20_cifar --lr 0.1 -p 50 -b 128 -j 1 --epochs 200 --compress=../quantization/quant_aware_train/preact_resnet_cifar_dorefa.yaml --wd=0.0002 --vs=0 --gpus 0", for your reference.
I tried to add quantization noise, and accuracy was fine. But in my opinion it is not perfect emulation for quantization because data distribution is not uniform.
i am trying to plot an current-voltage acquisition from an instrument but for different temperatures. I need to keep the structure as presented but i would like to have for each temperature get the plot in 1 uniform color then for the next temperature the color is changed so i can identify different plot and read their legend (the temperature). right now i get the animation but i cannot iterate on temperatures and colors. thanks
import random
import csv
from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
import pandas as pd
class IV_SWEEP():
def __init__(self):
self.IV_sweep()
def init_instrument(self):
#######settings
self.data_points = 10 # number of steps in the sweep
self.max_current = 0.001
self.min_current = -self.max_current
def measurement(self):
# Allocate arrays to store the measurement results
currents = np.linspace(self.min_current, self.max_current, num=self.data_points)
voltages = np.zeros_like(currents)
temp_list = [300, 310, 320] # want to plot a curve for each of this temperature
for i in range(self.data_points):
voltages[i] = random.random() + currents[i]
now = datetime.now()
self.record_csv(now, currents[i], voltages[i])
self.animate()
self.plot() # i am calling the plot method separately but dont know how to iterate the label so the color and legend is changed per temp
plt.pause(3)
self.plot_enable = True
def IV_sweep(self):
self.xdata = []
self.ydata = []
self.init_instrument()
self.measurement()
def animate(self):
data = pd.read_csv("test.csv")
self.xdata = data["Voltage (V)"]
self.ydata = data["Current (A)"]
def plot(self):
plt.gcf()
temp_list = [300, 310, 320]
labels = temp_list
# for lab in labels:
plt.scatter(self.xdata, self.ydata, label=temp_list) # need to add a different lab for each temperature scan
plt.xlabel('Voltage (V)')
plt.ylabel('Current (mA)')
# plt.title("IV sweep at {} K".format(temp_list[i]))
plt.title("IV sweep at temperature xx") # call the temperature so it s printed as legend
# plt.legend(loc='best')
plt.tight_layout()
return
def record_csv(filename,timestamp, currents, voltages):
filename = "test.csv"
with open(filename, 'a', newline='') as csvfile:
header = ["Timestamp", "Current (A)", "Voltage (V)", "Voltage stdv (V)"]
writer = csv.DictWriter(csvfile, fieldnames=header)
if csvfile.tell() == 0:
writer.writeheader()
writer.writerow(
{
"Timestamp": timestamp,
"Current (A)": currents,
"Voltage (V)": voltages,
}
)
csvfile.close()
IV_SWEEP()
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import random
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)
def main(i):
# Actual parameters
# A0 = 10
A0 = 2
K0 = [1, 3, 5]
C0 = 5
# Generate some data based on these
tmin, tmax = 0, 0.5
num = 20
t = []
y = []
for K in K0: #would like to scan on K and plot each curve with a different color
t = np.linspace(tmin, tmax, num)
y = model_func(t, A0, K, C0)
ax1.scatter(t,y, label = K) # would like to have as a legend each K
ax1.legend(loc="upper left")
def model_func(t, A, K, C):
return A * np.exp(K * t) + C
ani = animation.FuncAnimation(fig, main, interval=1000)
plt.show()
Result:
Tensorflow: convert PrefetchDataset to BatchDataset
With latest Tensorflow version 2.3.1I am trying to follow basic text classification example at: https://www.tensorflow.org/tutorials/keras/text_classification. Instead of creating dataset from directory as example does, I am using a csv file:
SELECT_COLUMNS = ['SentimentText','Sentiment']
LABEL_COLUMN = 'Sentiment'
LABELS = [0, 1]
def get_dataset(file_path, **kwargs):
dataset = tf.data.experimental.make_csv_dataset(
file_path,
batch_size=3, # Artificially small to make examples easier to show.
label_name=LABEL_COLUMN,
na_value="?",
num_epochs=1,
ignore_errors=True,
**kwargs)
return dataset
all_data = get_dataset(data_path, select_columns=SELECT_COLUMNS)
As a result I get:
type(all_data)
tensorflow.python.data.ops.dataset_ops.PrefetchDataset
Example loads data from directory with:
batch_size = 32
seed = 42
raw_train_ds = tf.keras.preprocessing.text_dataset_from_directory(
'aclImdb/train',
batch_size=batch_size,
validation_split=0.2,
subset='training',
seed=seed)
And gets dataset of another type:
type(raw_train_ds)
tensorflow.python.data.ops.dataset_ops.BatchDataset
Now when I try to standardise and vectorise data with functions from example:
def custom_standardization(input_data):
lowercase = tf.strings.lower(input_data)
stripped_html = tf.strings.regex_replace(lowercase, '<br />', ' ')
return tf.strings.regex_replace(stripped_html,
'[%s]' % re.escape(string.punctuation),
'')
max_features = 10000
sequence_length = 250
vectorize_layer = TextVectorization(
standardize=custom_standardization,
max_tokens=max_features,
output_mode='int',
output_sequence_length=sequence_length)
And apply them to my dataset I get error:
# Make a text-only dataset (without labels), then call adapt
train_text = all_data.map(lambda x, y: x)
vectorize_layer.adapt(train_text)
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-20-1f1fc445912d> in <module>
1 # Make a text-only dataset (without labels), then call adapt
2 train_text = all_data.map(lambda x, y: x)
----> 3 vectorize_layer.adapt(train_text)
/opt/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/layers/preprocessing/text_vectorization.py in adapt(self, data, reset_state)
378 shape = dataset_ops.get_legacy_output_shapes(data)
379 if not isinstance(shape, tensor_shape.TensorShape):
--> 380 raise ValueError("The dataset passed to 'adapt' must contain a single "
381 "tensor value.")
382 if shape.rank == 0:
ValueError: The dataset passed to 'adapt' must contain a single tensor value.
How to convert PrefetchDataset to BatchDataset ?
You could use tf.stack method to pack the features into a single array. The below function is from Custom training: walkthrough in Tensorflow.
def pack_features_vector(features, labels):
features = tf.stack(list(features.values()), axis=1)
return features, labels
all_data = get_dataset(data_path, select_columns=SELECT_COLUMNS)
train_dataset = all_data.map(pack_features_vector)
train_text = train_dataset.map(lambda x, y: x)
vectorize_layer.adapt(train_text)
I have (in a larger project) data contained in numpy.array.
Based on user input I need to move a selected axis (dimAxisNr) to the first dimension of the array and slice one or more (including the first) dimension based on user input (such as Select2 and Select0 in the example).
Using this input I generate a DataSelect which contains the information needed to slice. But the output size of the sliced array is different from the one using inline indexing. So basically I need a way to generate the '37:40:2' and '0:2' from an input list.
import numpy as np
dimAxisNr = 1
Select2 = [37,39]
Select0 = [0,1]
plotData = np.random.random((102,72,145,2))
DataSetSize = np.shape(plotData)
DataSelect = [slice(0,item) for item in DataSetSize]
DataSelect[2] = np.array(Select2)
DataSelect[0] = np.array(Select0)
def shift(seq, n):
n = n % len(seq)
return seq[n:] + seq[:n]
#Sort and Slice the data
print(np.shape(plotData))
print(DataSelect)
plotData = np.transpose(plotData, np.roll(range(plotData.ndim),-dimAxisNr))
DataSelect = shift(DataSelect,dimAxisNr)
print(DataSelect)
print(np.shape(plotData))
plotData = plotData[DataSelect]
print(np.shape(plotData))
plotDataDirect = plotData[slice(0, 72, None), 37:40:2, slice(0, 2, None), 0:2]
print(np.shape(plotDataDirect))
I'm not sure I've understood your question at all...
But if the question is "How do I generate a slice based on a list of indices like [37,39,40,23] ?"
then I would answer : you don't have to, just use the list as is to select the right indices, like so :
a = np.random.rand(4,5)
print(a)
indices = [2,3,1]
print(a[0:2,indices])
Note that the sorting of the list matters: [2,3,1] yields a different result from [1,2,3]
Output :
>>> a
array([[ 0.47814802, 0.42069094, 0.96244966, 0.23886243, 0.86159478],
[ 0.09248812, 0.85569145, 0.63619014, 0.65814667, 0.45387509],
[ 0.25933109, 0.84525826, 0.31608609, 0.99326598, 0.40698516],
[ 0.20685221, 0.1415642 , 0.21723372, 0.62213483, 0.28025124]])
>>> a[0:2,[2,3,1]]
array([[ 0.96244966, 0.23886243, 0.42069094],
[ 0.63619014, 0.65814667, 0.85569145]])
I have found the answer to my question. I need to use numpy.ix_.
Here is the working code:
import numpy as np
dimAxisNr = 1
Select2 = [37,39]
Select0 = [0,1]
plotData = np.random.random((102,72,145,2))
DataSetSize = np.shape(plotData)
DataSelect = [np.arange(0,item) for item in DataSetSize]
DataSelect[2] = Select2
DataSelect[0] = Select0
#print(list(37:40:2))
def shift(seq, n):
n = n % len(seq)
return seq[n:] + seq[:n]
#Sort and Slice the data
print(np.shape(plotData))
print(DataSelect)
plotData = np.transpose(plotData, np.roll(range(plotData.ndim),-dimAxisNr))
DataSelect = shift(DataSelect,dimAxisNr)
plotDataSlice = plotData[np.ix_(*DataSelect)]
print(np.shape(plotDataSlice))
plotDataDirect = plotData[slice(0, 72, None), 37:40:2, slice(0, 2, None), 0:1]
print(np.shape(plotDataDirect))
I am setting up an optimization in OpenMDAO v0.13 using several components that are used many times. My assembly seems to be working just fine with the default driver, but when I run with an optimizer it does not solve. The optimizer simply runs with the inputs given and returns the answer using those inputs. I am not sure what the issue is, but I would appreciate any insights. I have included a simple code mimicking my structure that reproduces the error. I think the problem is in the connections, summer.fs does not update after initialization.
from openmdao.main.api import Assembly, Component
from openmdao.lib.datatypes.api import Float, Array, List
from openmdao.lib.drivers.api import DOEdriver, SLSQPdriver, COBYLAdriver, CaseIteratorDriver
from pyopt_driver.pyopt_driver import pyOptDriver
import numpy as np
class component1(Component):
x = Float(iotype='in')
y = Float(iotype='in')
term1 = Float(iotype='out')
a = Float(iotype='in', default_value=1)
def execute(self):
x = self.x
a = self.a
term1 = a*x**2
self.term1 = term1
print "In comp1", self.name, self.a, self.x, self.term1
def list_deriv_vars(self):
return ('x',), ('term1',)
def provideJ(self):
x = self.x
a = self.a
dterm1_dx = 2.*a*x
J = np.array([[dterm1_dx]])
print 'In comp1, J = %s' % J
return J
class component2(Component):
x = Float(iotype='in')
y = Float(iotype='in')
term1 = Float(iotype='in')
f = Float(iotype='out')
def execute(self):
y = self.y
x = self.x
term1 = self.term1
f = term1 + x + y**2
self.f = f
print "In comp2", self.name, self.x, self.y, self.term1, self.f
class summer(Component):
total = Float(iotype='out', desc='sum of all f values')
def __init__(self, size):
super(summer, self).__init__()
self.size = size
self.add('fs', Array(np.ones(size), iotype='in', desc='f values from all cases'))
def execute(self):
self.total = sum(self.fs)
print 'In summer, fs = %s and total = %s' % (self.fs, self.total)
class assembly(Assembly):
x = Float(iotype='in')
y = Float(iotype='in')
total = Float(iotype='out')
def __init__(self, size):
super(assembly, self).__init__()
self.size = size
self.add('a_vals', Array(np.zeros(size), iotype='in', dtype='float'))
self.add('fs', Array(np.zeros(size), iotype='out', dtype='float'))
print 'in init a_vals = %s' % self.a_vals
def configure(self):
# self.add('driver', SLSQPdriver())
self.add('driver', pyOptDriver())
self.driver.optimizer = 'SNOPT'
# self.driver.pyopt_diff = True
#create this first, so we can connect to it
self.add('summer', summer(size=len(self.a_vals)))
self.connect('summer.total', 'total')
print 'in configure a_vals = %s' % self.a_vals
# create instances of components
for i in range(0, self.size):
c1 = self.add('comp1_%d'%i, component1())
c1.missing_deriv_policy = 'assume_zero'
c2 = self.add('comp2_%d'%i, component2())
self.connect('a_vals[%d]' % i, 'comp1_%d.a' % i)
self.connect('x', ['comp1_%d.x'%i, 'comp2_%d.x'%i])
self.connect('y', ['comp1_%d.y'%i, 'comp2_%d.y'%i])
self.connect('comp1_%d.term1'%i, 'comp2_%d.term1'%i)
self.connect('comp2_%d.f'%i, 'summer.fs[%d]'%i)
self.driver.workflow.add(['comp1_%d'%i, 'comp2_%d'%i])
self.connect('summer.fs[:]', 'fs[:]')
self.driver.workflow.add(['summer'])
# set up main driver (optimizer)
self.driver.iprint = 1
self.driver.maxiter = 100
self.driver.accuracy = 1.0e-6
self.driver.add_parameter('x', low=-5., high=5.)
self.driver.add_parameter('y', low=-5., high=5.)
self.driver.add_objective('summer.total')
if __name__ == "__main__":
""" the result should be -1 at (x, y) = (-0.5, 0) """
import time
from openmdao.main.api import set_as_top
a_vals = np.array([1., 1., 1., 1.])
test = set_as_top(assembly(size=len(a_vals)))
test.a_vals = a_vals
print test.a_vals
test.x = 2.
test.y = 2.
tt = time.time()
test.run()
print "Elapsed time: ", time.time()-tt, "seconds"
print 'result = ', test.summer.total
print '(x, y) = (%s, %s)' % (test.x, test.y)
print test.fs
I played around with your model, and found that the following line caused problems:
#self.connect('summer.fs[:]', 'fs[:]')
When I commented it out, I got the optimization to move.
I am not sure what is happening there, but the graph transformations sometimes have some issues with component input nodes that are promoted as outputs on the assembly boundary. If you still want those values to be available on the assembly, you could try promoting the outputs from the comp2_n components instead.