Fusion of vgg19 model and dense layer model - concatenation

Question

AIM: To perform multiclassification of materials using images and roughness values.
DATASET - I have a dataset which has 11 material classes, each class has 25 images. So the total number of train images are 2011=220, and 511=55 belong to validation set. Each image has a corresponding 6 roughness parameters. These parameters are stored in csv file as input = RaX, RaY, RqX, RqY, RzX, RzY and output = 11 material classes. So for every class, 25 values of RaX, RaY, RqX, RqY, RzX, RzY are recorded. I have trained the images on vgg19 and roughness values as dense layers.
PROBLEM STATEMENT - The training accuracy is 7% and loss is 0.7. The vgg model and dense model when trained separately gave good scores. But fusion or concatenation of both models reduced the scores significantly.
Could you please help me out in this regard. Any suggestions and information are welcomed. Thanks in advance.
Below is the description of the code:
data_path = "Excel_data/Roughness_11_classes.csv"
image_path = "WOOD_PLASTIC/IMAGES_11_classes"
classes=11
rv=[]#### return values object list from rough_data values
label=[]
image_list = []
a=[]
batch_size = 16
vgg19 = applications.VGG19(include_top=False, weights='imagenet')
datagen = ImageDataGenerator(rescale=1. / 255,validation_split=0.2)
img_width, img_height = 128, 512
def rough_values(path): ## process rougness values
global trainRX
data = pd.read_csv(data_path)
data_array=data.to_numpy()
x=data_array[:,0:6].astype(np.float32)
y1=data_array[:,6].reshape(-1,1)
encoder = OneHotEncoder(sparse=False)
y = encoder.fit_transform(y1)
trainRX,testRX,trainRY,testRY=train_test_split(x,y,test_size=0.2,shuffle=False,random_state=0)
rv=[data_array,trainRX,testRX,trainRY,testRY]
return rv
def images(path): ### processes images
##### TRAIN #########################################################
generator_train = datagen.flow_from_directory(path, target_size=(img_width,
img_height),batch_size=batch_size, class_mode='categorical', shuffle=False,
subset='training')
nb_train_samples = len(generator_train.filenames)
num_classes = len(generator_train.class_indices)
predict_size_train = int(math.ceil(nb_train_samples / batch_size))
bottleneck_features_train = vgg19.predict_generator(generator_train,
predict_size_train)
print (bottleneck_features_train.shape)
np.save("bottleneck_features_vgg19_multi_input.npy", bottleneck_features_train)
#####saves train data as .npy file
#VALIDATION##################################################################################
generator_val = datagen.flow_from_directory(path,target_size=(img_width, img_height),batch_size=batch_size, class_mode='categorical', shuffle=False, subset='validation')
print('generator_val is',generator_val)
nb_validation_samples = len(generator_val.filenames)
num_classes = len(generator_val.class_indices)
predict_size_validation = int(math.ceil(nb_validation_samples / batch_size))
bottleneck_features_validation = vgg19.predict_generator(generator_val, predict_size_validation)
np.save("bottleneck_features_validation_vgg19_multi_input.npy", bottleneck_features_validation)
# training data load #############################################################
generator_top_train = datagen.flow_from_directory(path,target_size=(img_width, img_height),batch_size=batch_size, class_mode='categorical',shuffle=False,subset='training')
nb_train_samples = len(generator_top_train.filenames)
print('nb_train_samples are',nb_train_samples)
num_classes = len(generator_top_train.class_indices)
train_data = np.load("bottleneck_features_vgg19_multi_input.npy")
print (train_data.shape)
train_labels = generator_top_train.classes
train_labels = to_categorical(train_labels, num_classes=num_classes)
# validation data load #############################################################
generator_top_val = datagen.flow_from_directory(path,target_size=(img_width, img_height),batch_size=batch_size, class_mode='categorical',shuffle=False,subset='validation')
nb_validation_samples = len(generator_top_val.filenames)
print('nb_validation_samples are',nb_validation_samples)
num_classes = len(generator_top_val.class_indices)
validation_data = np.load("bottleneck_features_validation_vgg19_multi_input.npy")
validation_labels = generator_top_val.classes
validation_labels = to_categorical(validation_labels, num_classes=num_classes)
z=[train_data, validation_data, validation_labels, train_labels]
return z
def create_dense(feed): ## cnn model for roughness parameters, define MLP network
model = Sequential() ### input shape to dense is 1D
model.add(Dense(64, input_shape=feed, activation='relu', name='fc1'))
model.add(BatchNormalization())
model.add(Dense(32, activation='relu', name='fc2'))
model.add(BatchNormalization())
return model
def create_vgg(input_shape, n_classes, optimizer='rmsprop', fine_tune=0):
### cnn model for images
######### Adding own model on top of vgg ##################
model = Sequential()
model.add(Flatten(input_shape=input_shape))
model.add(Dropout(0.5))
model.add(Dense(100))
model.add(layers.LeakyReLU(alpha=0.3))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(50))
model.add(layers.LeakyReLU(alpha=0.3))
model.add(BatchNormalization())
model.add(Dropout(0.5))
### Group the convolutional base and new fully-connected layers into a Model object#
return model
rv=rough_values(data_path)
input_dense=[]
DA=rv[0]
TrRX=rv[1]
TeRX=rv[2]
TrRY=rv[3]
TeRY=rv[4]
z=images(image_path) ## stores train_data, validation_data, validation_labels,
train_labels
train_data=z[0]
vali=z[1]
val_label=z[2]
tr_label=z[3]
input_dense=TrRX[0].shape
class_names=
['ABS','PA','PC','PP','WOOD1','WOOD2','WOOD3','WOOD4','WOOD5','WOOD6','WOOD7']
### 11 material classes
mlp = create_dense(input_dense)
cnn = create_vgg(train_data.shape[1:],classes)
combinedInput = concatenate([mlp.output, cnn.output]) ## concatenated models
top= Dense(64, activation="relu")(combinedInput)
top_model= Dense(11, activation="softmax")(top) ### output as 11 classes
model = Model(inputs=[mlp.input, cnn.input], outputs=top_model)
## training #################################################################
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=optimizers.Adam(lr=1e-4),metrics=['acc'])
history= model.fit(x=[TrRX, train_data], y=TrRY, validation_data=([TeRX, vali], TeRY),
epochs=100, batch_size=32,validation_steps=(55//batch_size))
(eval_loss, eval_accuracy) = model.evaluate([TeRX, vali], TeRY, batch_size=batch_size,
verbose=1)
print("test accuracy: {:.2f}%".format(eval_accuracy * 100))
print("test Loss: {}".format(eval_loss))
model.save('fusion_model.h5')
#result=model.predict([TeRX,vali])