Odtnhj

I am trying to train a U-net for image segmentation on satellite data and therewith extract a road network with nine different road types. Thus far I have tried many different U-net codes that are freely available on the web, however I was not able to tailor them to my specific case. I'm sincerely hoping you are able to help me.

The satellite image and associated labels can be downloaded via the following link:
Satellite image and associated labels

Additionally, I've written the following code to prep the data for the Unet

import skimage
from skimage.io import imread, imshow, imread_collection, concatenate_images
from skimage.transform import resize
from skimage.morphology import label
import numpy as np
import matplotlib.pyplot as plt
from keras.models import Model
from keras.layers import Input, merge, Convolution2D, MaxPooling2D, UpSampling2D, Reshape, core, Dropout
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, LearningRateScheduler
from keras import backend as K
from sklearn.metrics import jaccard_similarity_score
from shapely.geometry import MultiPolygon, Polygon
import shapely.wkt
import shapely.affinity
from collections import defaultdict

#Importing image and labels
labels = skimage.io.imread("ede_subset_293_wegen.tif")
images = skimage.io.imread("ede_subset_293_20180502_planetscope.tif")[...,:-1]

#Scaling image
img_scaled = images / images.max()

#Make non-roads 0
labels[labels == 15] = 0

#Resizing image and mask and labels
img_scaled_resized = img_scaled[:6400, :6400,:4 ]
print(img_scaled_resized.shape)
labels_resized = labels[:6400, :6400]
print(labels_resized.shape)

#splitting images
split_img = [
 np.split(array, 25, axis=0) 
 for array in np.split(img_scaled_resized, 25, axis=1)
]

split_img[-1][-1].shape

#splitting labels
split_labels = [
 np.split(array, 25, axis=0) 
 for array in np.split(labels_resized, 25, axis=1)
]

#Convert to np.array
split_labels = np.array(split_labels)
split_img = np.array(split_img)

train_images = np.reshape(split_img, (625, 256, 256, 4))
train_labels = np.reshape(split_labels, (625, 256, 256))

x_trn = train_images[:400,:,:,:]
x_val = train_images[400:500,:,:,:]
x_test = train_images[500:625,:,:,:]

y_trn = train_labels[:400,:,:]
y_val = train_labels[400:500,:,:]
y_test = train_labels[500:625,:,:]

plt.imshow(train_images[88,:,:,:])
skimage.io.imshow(train_labels[88,:,:])

Furthermore, I found the following U-net on kaggle, which I think should have to work for this particular case:

def get_unet():
 inputs = Input((8, ISZ, ISZ))
 conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(inputs)
 conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv1)
 pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)

 conv2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(pool1)
 conv2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv2)
 pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)

 conv3 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(pool2)
 conv3 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv3)
 pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)

 conv4 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(pool3)
 conv4 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(conv4)
 pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)

 conv5 = Convolution2D(512, 3, 3, activation='relu', border_mode='same')(pool4)
 conv5 = Convolution2D(512, 3, 3, activation='relu', border_mode='same')(conv5)

 up6 = merge([UpSampling2D(size=(2, 2))(conv5), conv4], mode='concat', concat_axis=1)
 conv6 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(up6)
 conv6 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(conv6)

 up7 = merge([UpSampling2D(size=(2, 2))(conv6), conv3], mode='concat', concat_axis=1)
 conv7 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(up7)
 conv7 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv7)

 up8 = merge([UpSampling2D(size=(2, 2))(conv7), conv2], mode='concat', concat_axis=1)
 conv8 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(up8)
 conv8 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv8)

 up9 = merge([UpSampling2D(size=(2, 2))(conv8), conv1], mode='concat', concat_axis=1)
 conv9 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(up9)
 conv9 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv9)

 conv10 = Convolution2D(N_Cls, 1, 1, activation='sigmoid')(conv9)

 model = Model(input=inputs, output=conv10)
 model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=[jaccard_coef, jaccard_coef_int, 'accuracy'])
 return model

I know it is a big question, but I'm getting pretty desperate. Any help is greatly appreciated!

Kind regards,

Eeuwigestudent1

i found that Conv2DTranspose works better than UpSampling2D and here is a quick implementation using the same

def conv_block(tensor, nfilters, size=3, padding='same', initializer="he_normal"):
 x = Conv2D(filters=nfilters, kernel_size=(size, size), padding=padding, kernel_initializer=initializer)(tensor)
 x = BatchNormalization()(x)
 x = Activation("relu")(x)
 x = Conv2D(filters=nfilters, kernel_size=(size, size), padding=padding, kernel_initializer=initializer)(x)
 x = BatchNormalization()(x)
 x = Activation("relu")(x)
 return x


def deconv_block(tensor, residual, nfilters, size=3, padding='same', strides=(2, 2)):
 y = Conv2DTranspose(nfilters, kernel_size=(size, size), strides=strides, padding=padding)(tensor)
 y = concatenate([y, residual], axis=3)
 y = conv_block(y, nfilters)
 return y


def Unet(img_height, img_width, nclasses=3, filters=64):
# down
 input_layer = Input(shape=(img_height, img_width, 3), name='image_input')
 conv1 = conv_block(input_layer, nfilters=filters)
 conv1_out = MaxPooling2D(pool_size=(2, 2))(conv1)
 conv2 = conv_block(conv1_out, nfilters=filters*2)
 conv2_out = MaxPooling2D(pool_size=(2, 2))(conv2)
 conv3 = conv_block(conv2_out, nfilters=filters*4)
 conv3_out = MaxPooling2D(pool_size=(2, 2))(conv3)
 conv4 = conv_block(conv3_out, nfilters=filters*8)
 conv4_out = MaxPooling2D(pool_size=(2, 2))(conv4)
 conv4_out = Dropout(0.5)(conv4_out)
 conv5 = conv_block(conv4_out, nfilters=filters*16)
 conv5 = Dropout(0.5)(conv5)
# up
 deconv6 = deconv_block(conv5, residual=conv4, nfilters=filters*8)
 deconv6 = Dropout(0.5)(deconv6)
 deconv7 = deconv_block(deconv6, residual=conv3, nfilters=filters*4)
 deconv7 = Dropout(0.5)(deconv7) 
 deconv8 = deconv_block(deconv7, residual=conv2, nfilters=filters*2)
 deconv9 = deconv_block(deconv8, residual=conv1, nfilters=filters)
# output
 output_layer = Conv2D(filters=nclasses, kernel_size=(1, 1))(deconv9)
 output_layer = BatchNormalization()(output_layer)
 output_layer = Activation('softmax')(output_layer)

 model = Model(inputs=input_layer, outputs=output_layer, name='Unet')
 return model

Now for the data generators, you can use the builtin ImageDataGenerator class
here is the code from Keras docs

# we create two instances with the same arguments
data_gen_args = dict(featurewise_center=True,
 featurewise_std_normalization=True,
 rotation_range=90,
 width_shift_range=0.1,
 height_shift_range=0.1,
 zoom_range=0.2)
image_datagen = ImageDataGenerator(**data_gen_args)
mask_datagen = ImageDataGenerator(**data_gen_args)

# Provide the same seed and keyword arguments to the fit and flow methods
seed = 1
image_datagen.fit(images, augment=True, seed=seed)
mask_datagen.fit(masks, augment=True, seed=seed)

image_generator = image_datagen.flow_from_directory(
 'data/images',
 class_mode=None,
 seed=seed)

mask_generator = mask_datagen.flow_from_directory(
 'data/masks',
 class_mode=None,
 seed=seed)

# combine generators into one which yields image and masks
train_generator = zip(image_generator, mask_generator)

model.fit_generator(
 train_generator,
 steps_per_epoch=2000,
 epochs=50)

Another way to go is implement your own generator by extending the Sequence class from Keras

class seg_gen(Sequence):
 def __init__(self, x_set, y_set, batch_size, image_dir, mask_dir):
 self.x, self.y = x_set, y_set
 self.batch_size = batch_size
 self.samples = len(self.x)
 self.image_dir = image_dir
 self.mask_dir = mask_dir

 def __len__(self):
 return int(np.ceil(len(self.x) / float(self.batch_size)))

 def __getitem__(self, idx):
 idx = np.random.randint(0, self.samples, batch_size)
 batch_x, batch_y = , 
 drawn = 0
 for i in idx:
 _image = image.img_to_array(image.load_img(f'self.image_dir/self.x[i]', target_size=(img_height, img_width)))/255. 
 mask = image.img_to_array(image.load_img(f'self.mask_dir/self.y[i]', grayscale=True, target_size=(img_height, img_width)))
# mask = np.resize(mask,(img_height*img_width, classes))
 batch_y.append(mask)
 batch_x.append(_image)
 return np.array(batch_x), np.array(batch_y)

Here is a sample code to train the model

unet = Unet(256, 256, nclasses=66, filters=64)
print(unet.output_shape)
p_unet = multi_gpu_model(unet, 4)
p_unet.load_weights('models-dr/top_weights.h5')
p_unet.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
tb = TensorBoard(log_dir='logs', write_graph=True)
mc = ModelCheckpoint(mode='max', filepath='models-dr/top_weights.h5', monitor='acc', save_best_only='True', save_weights_only='True', verbose=1)
es = EarlyStopping(mode='max', monitor='acc', patience=6, verbose=1)
callbacks = [tb, mc, es]
train_gen = seg_gen(image_list, mask_list, batch_size)


p_unet.fit_generator(train_gen, steps_per_epoch=steps, epochs=13, callbacks=callbacks, workers=8)

I have tried using the dice loss when i had only two classes, here is the code for it

def dice_coeff(y_true, y_pred):
 smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
 return score

def dice_loss(y_true, y_pred):
 loss = 1 - dice_coeff(y_true, y_pred)
 return loss