I'm using MONAI on Spyder Anaconda to build a U-Net network. I want to add/modify layers starting from this baseline.
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = nets.UNet(
spatial_dims = 2,
in_channels = 3,
out_channels = 1,
channels = (4, 8, 16, 32, 64),
strides = (2, 2, 2, 2),
num_res_units = 3,
norm = layers.Norm.BATCH,
kernel_size=3,).to(device)
loss_function = losses.DiceLoss()
torch.backends.cudnn.benchmark = True
optimizer = torch.optim.Adam(model.parameters(), lr = 1e-4, weight_decay = 0)
post_pred = Compose([EnsureType(), Activations(sigmoid = True), AsDiscrete(threshold=0.5)])
post_label = Compose([EnsureType()])
inferer = SimpleInferer()
utils.set_determinism(seed=46)
My final aim is to create a MultiResUNet that has different layers such as:
class Conv2d_batchnorm(torch.nn.Module):
'''
2D Convolutional layers
Arguments:
num_in_filters {int} -- number of input filters
num_out_filters {int} -- number of output filters
kernel_size {tuple} -- size of the convolving kernel
stride {tuple} -- stride of the convolution (default: {(1, 1)})
activation {str} -- activation function (default: {'relu'})
'''
def __init__(self, num_in_filters, num_out_filters, kernel_size, stride = (1,1), activation = 'relu'):
super().__init__()
self.activation = activation
self.conv1 = torch.nn.Conv2d(in_channels=num_in_filters, out_channels=num_out_filters, kernel_size=kernel_size, stride=stride, padding = 'same')
self.batchnorm = torch.nn.BatchNorm2d(num_out_filters)
def forward(self,x):
x = self.conv1(x)
x = self.batchnorm(x)
if self.activation == 'relu':
return torch.nn.functional.relu(x)
else:
return x
This is just an example of a different Conv2d layer that I would use instead of the native one of the baseline.
Hope some of you can figure out how to proceed.
Thanks, Fede
Related
I modified Distiller (https://github.com/IntelLabs/distiller) to emulate in-memory computing circuit, especially added a convolution layer quantization during QAT. However, accuracy drops over 60% (90% > 30%) even with 32b quantization for sanity check. I also want to say that it was perfectly fine when I just add a calculated quantization noise. Below is the code.
import torch
import torch.nn as nn
import math
__all__ = ['preact_resnet20_cifar', 'preact_resnet32_cifar', 'preact_resnet44_cifar', 'preact_resnet56_cifar',
'preact_resnet110_cifar', 'preact_resnet20_cifar_conv_ds', 'preact_resnet32_cifar_conv_ds',
'preact_resnet44_cifar_conv_ds', 'preact_resnet56_cifar_conv_ds', 'preact_resnet110_cifar_conv_ds']
NUM_CLASSES = 10
device = torch.device("cuda")
def conv3x3(in_planes, out_planes, stride=1):
"""3x3 convolution with padding"""
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
def quantize(tensor, n_bits=32, dim=144, clip_ratio=1):
max_value = dim*clip_ratio
min_value = -dim*clip_ratio
delta = max_value - min_value
y = tensor.clone()
y = torch.clamp(y, min = min_value, max = max_value)
lsb = delta / (2**n_bits)
y = (y // lsb)*lsb
return y
class PreactBasicBlock(nn.Module):
expansion = 1
def __init__(self, block_gates, inplanes, planes, stride=1, downsample=None, preact_downsample=True, resolution = 32):
super(PreactBasicBlock, self).__init__()
self.block_gates = block_gates
self.pre_bn = nn.BatchNorm2d(inplanes)
self.pre_relu = nn.ReLU(inplace=False) # To enable layer removal inplace must be False
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn = nn.BatchNorm2d(planes)
self.relu = nn.ReLU(inplace=False)
self.conv2 = conv3x3(planes, planes)
self.downsample = downsample
self.stride = stride
self.preact_downsample = preact_downsample
self.resolution = resolution
def forward(self, x):
need_preact = self.block_gates[0] or self.block_gates[1] or self.downsample and self.preact_downsample # add pre-activation for block_gates 0 and 1 and when is downsampled
if need_preact: # x > bn > relu > out
preact = self.pre_bn(x)
preact = self.pre_relu(preact)
out = preact
else: # x > out
preact = out = x
if self.block_gates[0]: # (preact) > conv > bn > relu
out = self.conv1(out)
dim = self.conv1.in_channels * self.conv1.kernel_size[0] * self.conv1.kernel_size[1]
out = quantize(out, n_bits=self.resolution, dim=dim, clip_ratio=1)
out = self.bn(out)
out = self.relu(out)
if self.block_gates[1]: # (preact)> conv
out = self.conv2(out)
dim = self.conv2.in_channels * self.conv2.kernel_size[0] * self.conv2.kernel_size[1]
out = quantize(out, n_bits=self.resolution, dim=dim, clip_ratio=1)
if self.downsample is not None:
if self.preact_downsample:
residual = self.downsample(preact)
else:
residual = self.downsample(x)
else:
residual = x
out += residual
return out
class PreactResNetCifar(nn.Module):
def __init__(self, block, layers, num_classes=NUM_CLASSES, conv_downsample=False):
self.nlayers = 0
# Each layer manages its own gates
self.layer_gates = []
for layer in range(3):
# For each of the 3 layers, create block gates: each block has two layers
self.layer_gates.append([]) # [True, True] * layers[layer])
for blk in range(layers[layer]):
self.layer_gates[layer].append([True, True])
self.inplanes = 16 # 64
super(PreactResNetCifar, self).__init__()
self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
self.layer1 = self._make_layer(self.layer_gates[0], block, 16, layers[0],
conv_downsample=conv_downsample)
self.layer2 = self._make_layer(self.layer_gates[1], block, 32, layers[1], stride=2,
conv_downsample=conv_downsample)
self.layer3 = self._make_layer(self.layer_gates[2], block, 64, layers[2], stride=2,
conv_downsample=conv_downsample)
self.final_bn = nn.BatchNorm2d(64 * block.expansion)
self.final_relu = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(8, stride=1)
self.fc = nn.Linear(64 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def _make_layer(self, layer_gates, block, planes, blocks, stride=1, conv_downsample=False):
downsample = None
outplanes = planes * block.expansion
if stride != 1 or self.inplanes != outplanes:
if conv_downsample:
downsample = nn.Conv2d(self.inplanes, outplanes,
kernel_size=1, stride=stride, bias=False)
else:
# Identity downsample uses strided average pooling + padding instead of convolution
pad_amount = int(self.inplanes / 2)
downsample = nn.Sequential(
nn.AvgPool2d(2),
nn.ConstantPad3d((0, 0, 0, 0, pad_amount, pad_amount), 0)
)
layers = []
layers.append(block(layer_gates[0], self.inplanes, planes, stride, downsample, conv_downsample))
self.inplanes = outplanes
for i in range(1, blocks):
layers.append(block(layer_gates[i], self.inplanes, planes))
return nn.Sequential(*layers)
def forward(self, x):
x = self.conv1(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.final_bn(x)
x = self.final_relu(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
return x
def preact_resnet20_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [3, 3, 3], **kwargs)
return model
def preact_resnet32_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [5, 5, 5], **kwargs)
return model
def preact_resnet44_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [7, 7, 7], **kwargs)
return model
def preact_resnet56_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [9, 9, 9], **kwargs)
return model
def preact_resnet110_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [18, 18, 18], **kwargs)
return model
def preact_resnet182_cifar(**kwargs):
model = PreactResNetCifar(PreactBasicBlock, [30, 30, 30], **kwargs)
return model
def preact_resnet20_cifar_conv_ds(**kwargs):
return preact_resnet20_cifar(conv_downsample=True)
def preact_resnet32_cifar_conv_ds(**kwargs):
return preact_resnet32_cifar(conv_downsample=True)
def preact_resnet44_cifar_conv_ds(**kwargs):
return preact_resnet44_cifar(conv_downsample=True)
def preact_resnet56_cifar_conv_ds(**kwargs):
return preact_resnet56_cifar(conv_downsample=True)
def preact_resnet110_cifar_conv_ds(**kwargs):
return preact_resnet110_cifar(conv_downsample=True)
def preact_resnet182_cifar_conv_ds(**kwargs):
return preact_resnet182_cifar(conv_downsample=True)
I use distiller.example.classifier_compression.compress_classifier.py in Distiller, and use terminal with schedule file. Command is "python compress_classifier.py -a preact_resnet20_cifar --lr 0.1 -p 50 -b 128 -j 1 --epochs 200 --compress=../quantization/quant_aware_train/preact_resnet_cifar_dorefa.yaml --wd=0.0002 --vs=0 --gpus 0", for your reference.
I tried to add quantization noise, and accuracy was fine. But in my opinion it is not perfect emulation for quantization because data distribution is not uniform.
I am trying to identify Global Feature Relationships with SHAP values. The SHAP library returns three matrices and I am trying to select the SHAP matrix however, I am getting this error: "IndexError: too many indices for array: array is 2-dimensional, but 3 were indexed".
The code I have is below:
df_score = spark.sql("select * from sandbox.yt_trng_churn_device")
#XGBoost Model
import pickle
from xgboost import XGBClassifier
from mlflow.tracking import MlflowClient
client = MlflowClient()
local_dir = "/dbfs/FileStore/"
local_path = client.download_artifacts
model_path = '/dbfs/FileStore/'
model = XGBClassifier()
model = pickle.load(open(model_path, 'rb'))
HorizonDate = datetime.datetime(2022, 9, 5)
df = df_score
score_data = df.toPandas()
results = model.predict_proba(score_data)
results_l = model.predict(score_data)
score_data["p"]=pd.Series( (v[1] for v in results) )
score_data["l"]=pd.Series( (v for v in results_l) )
spark.createDataFrame(score_data).createOrReplaceTempView("yt_vw_tmp_dev__scores")
spark.sql("create or replace table sandbox.yt_vw_tmp_dev__scores as select * from yt_vw_tmp_dev__scores")
#SHAP Analysis on XGBoost
from shap import KernelExplainer, summary_plot
sql = """
select d_a.*
from
hive_metastore.sandbox.yt_trng_device d_a
right join
(select decile, msisdn, MSISDN_L2L
from(
select ntile(10) over (order by p desc) as decile, msisdn, MSISDN_L2L
from sandbox.yt_vw_tmp_dev__scores
) inc
order by decile) d_b
on d_a.MSISDN_L2L = d_b.MSISDN_L2L and d_a.msisdn = d_b.msisdn
"""
df = spark.sql(sql).drop('msisdn', 'imei', 'imsi', 'event_date', 'MSISDN_L2L', 'account_id')
score_df = df.toPandas()
mode = score_df.mode().iloc[0]
sample = score_df.sample(n=min(100, score_df.shape[0]), random_state=508502835).fillna(mode)
predict = lambda x: model.predict(pd.DataFrame(x, columns=score_df.columns))
explainer = KernelExplainer(predict, sample, link="identity")
shap_values = explainer.shap_values(sample, l1_reg=False)
# The return of the explainer has three matrices, we will get the shap values one
shap_values = shap_values[ :, :, 0]
I am fairly new to coding but it would be great if someone could give some direction on this
I'm trying to load the Cornell dataset from PyTorch Geometric to train my Graph Neural Network. I want to apply a mask but I achieve this error (also on Chameleon, Wisconsin, Texas datasets). My Dataset class works perfectly with all the datasets of Planetoid that are mono dimensional tensors, probable bidimensional tensors give problem.
I insert my code that can be ruined on Colab without problems.
!pip install torch-scatter torch-sparse torch-cluster torch-spline-conv torch-geometric -f https://data.pyg.org/whl/torch-1.12.0+cu113.html
import torch_geometric
from torch_geometric.datasets import Planetoid, WebKB
from torch_geometric.utils import to_dense_adj, to_undirected, remove_self_loops
class Dataset(object):
def __init__(self, name):
super(Dataset, self).__init__()
self.name = name
if (name == 'Cora'):
dataset = Planetoid(root='/tmp/Cora', name='Cora', split="full")
if(name == 'Citeseer'):
dataset = Planetoid(root='/tmp/Cora', name='Citeseer', split="full")
if(name == 'PubMed'):
dataset = Planetoid(root='/tmp/Cora', name='Pubmed', split="full")
if(name == 'Cornell'):
dataset = WebKB(root='/tmp/WebKB', name='Cornell')
self.data = dataset[0]
print(self.data)
self.train_mask = self.data.train_mask
self.valid_mask = self.data.val_mask
self.test_mask = self.data.test_mask
def train_val_test_split(self):
train_x = self.data.x[self.data.train_mask]
train_y = self.data.y[self.data.train_mask]
valid_x = self.data.x[self.data.val_mask]
valid_y = self.data.y[self.data.val_mask]
test_x = self.data.x[self.data.test_mask]
test_y = self.data.y[self.data.test_mask]
return train_x, train_y, valid_x, valid_y, test_x, test_y
def get_fullx(self):
return self.data.x
def get_edge_index(self):
return self.data.edge_index
def get_adjacency_matrix(self):
# We will ignore this for the first part
adj = to_dense_adj(self.data.edge_index)[0]
return adj
The error that I achieve is in the title and is obtained in this snippet:
cornell_dataset = Dataset(name = 'Cornell')
train_x, train_y, valid_x, valid_y, test_x, test_y = cornell_dataset.train_val_test_split()
# check and confirm our data shapes match our expectations
print(f"Train shape x: {train_x.shape}, y: {train_y.shape}")
print(f"Val shape x: {valid_x.shape}, y: {valid_y.shape}")
print(f"Test shape x: {test_x.shape}, y: {test_y.shape}")
I'm trying to do a churn analysis with R and SQL Server 2016.
I have uploaded my dataset on my database in a local SQL Server and I did all the preliminary work on this dataset.
Well, now I have this function trainModel() which I would use to estimate my random model forest:
trainModel = function(sqlSettings, trainTable) {
sqlConnString = sqlSettings$connString
trainDataSQL <- RxSqlServerData(connectionString = sqlConnString,
table = trainTable,
colInfo = cdrColInfo)
## Create training formula
labelVar = "churn"
trainVars <- rxGetVarNames(trainDataSQL)
trainVars <- trainVars[!trainVars %in% c(labelVar)]
temp <- paste(c(labelVar, paste(trainVars, collapse = "+")), collapse = "~")
formula <- as.formula(temp)
## Train gradient tree boosting with mxFastTree on SQL data source
library(RevoScaleR)
rx_forest_model <- rxDForest(formula = formula,
data = trainDataSQL,
nTree = 8,
maxDepth = 16,
mTry = 2,
minBucket = 1,
replace = TRUE,
importance = TRUE,
seed = 8,
parms = list(loss = c(0, 4, 1, 0)))
return(rx_forest_model)
}
But when I run the function I get this wrong output:
> system.time({
+ trainModel(sqlSettings, trainTable)
+ })
user system elapsed
0.29 0.07 58.18
Warning message:
In tempGetNumObs(numObs) :
Number of observations not available for this data source. 'numObs' set to 1e6.
And for this warning message, the function trainModel() does not create the object rx_forest_model
Does anyone have any suggestions on how to solve this problem?
After several attempts, I found the reason why the function trainModel() did not function properly.
Is not a connection string problem and is not even a data source type issue.
The problem is in the syntax of function trainModel().
It is enough to eliminate from the body of the function the statement:
return(rx_forest_model)
In this way, the function returns the same warning message, but creates the object rx_forest_model in the correct way.
So, the correct function is:
trainModel = function(sqlSettings, trainTable) {
sqlConnString = sqlSettings$connString
trainDataSQL <- RxSqlServerData(connectionString = sqlConnString,
table = trainTable,
colInfo = cdrColInfo)
## Create training formula
labelVar = "churn"
trainVars <- rxGetVarNames(trainDataSQL)
trainVars <- trainVars[!trainVars %in% c(labelVar)]
temp <- paste(c(labelVar, paste(trainVars, collapse = "+")), collapse = "~")
formula <- as.formula(temp)
## Train gradient tree boosting with mxFastTree on SQL data source
library(RevoScaleR)
rx_forest_model <- rxDForest(formula = formula,
data = trainDataSQL,
nTree = 8,
maxDepth = 16,
mTry = 2,
minBucket = 1,
replace = TRUE,
importance = TRUE,
seed = 8,
parms = list(loss = c(0, 4, 1, 0)))
}
I am new to python and have tried to accomplish the following with only little success:
In a folder there are *.columns files that do all contain 5 (0-4) columns and 500 rows. I need to sum up the columns 1-4 over all *.columns files and plot the result against the first column of any of that (all equal).
I created an empty array in which I want to paste the first (0) column of array "x_array3" and the columns 1-4 from "y_array0". All of them have the same size (500L, 5L).
Could You please give me an advice how to proceed? I am lost right now.
Christian
import glob
import numpy as np
ListOfFiles = glob.glob("*.columns")
y_array0 = 0
for filename in ListOfFiles:
y_array1 = np.genfromtxt(filename, skip_header = 1, usecols = (0, 1, 2, 3, 4))
y_array0 = y_array0 + y_array1
x_array3 = np.genfromtxt(ListOfFiles[0], skip_header = 1, usecols = (0, 1, 2, 3, 4))
empty_array = np.empty(shape=(500, 5))
ausgabe_array = ??? here I'm stuck ???
np.savetxt('SX_DOS.out', ausgabe_array)
I have found a working solution. I read in all the columns as single arrays and merge them at the end. Still, can anybody give me a hint how one populates an empty array with selected items from another array (with another size)?
Chr.
import glob
import numpy as np
ListOfFiles = glob.glob("*.columns")
y_array_s0 = 0
y_array_p0 = 0
y_array_d0 = 0
y_array_f0 = 0
for filename in ListOfFiles:
y_array_s1 = np.genfromtxt(filename, skip_header = 1, usecols = (1))
y_array_s0 = y_array_s0 + y_array_s1
y_array_p1 = np.genfromtxt(filename, skip_header = 1, usecols = (2))
y_array_p0 = y_array_p0 + y_array_p1
y_array_d1 = np.genfromtxt(filename, skip_header = 1, usecols = (3))
y_array_d0 = y_array_d0 + y_array_d1
y_array_f1 = np.genfromtxt(filename, skip_header = 1, usecols = (4))
y_array_f0 = y_array_f0 + y_array_f1
x_array3 = np.genfromtxt(ListOfFiles[0], skip_header = 1, usecols = (0))
ausgabe_array = np.transpose(np.array((x_array3, y_array_s0, y_array_p0, y_array_d0,y_array_f0)))
np.savetxt('SX_DOS.out', ausgabe_array)
I think you are trying to sum columns in an imported array.
Assuming this is working:
y_array1 = np.genfromtxt(filename, skip_header = 1, usecols = (0, 1, 2, 3, 4))
then
y_array0 = y_array1.sum(axis=0)
should give the sum of the columns.
With regard to plotting, I would recommend matplotlib