Data Driven Modeling

(Theme of this semester: CODING AS LITERACY)


PhD seminar series at Chair for Computer Aided Architectural Design (CAAD), ETH Zurich

Vahid Moosavi


15th Session


11 April 2017

Introduction to Representation Learning: Convolutional Neural Networks

To be discussed

  • Review of Fourier and its limits
  • Feature Enginering and Feature (Representation) Learning
  • Convolution
  • Convolutional Neural Networks
  • Examples in Urban Studies
  • Extensions and applications
In [1]:
import warnings
warnings.filterwarnings("ignore")
import datetime
import pandas as pd
# import pandas.io.data
import numpy as np
import matplotlib
from matplotlib import pyplot as plt
import sys
import sompylib.sompy as SOM# from pandas import Series, DataFrame

from ipywidgets import interact, HTML, FloatSlider

%matplotlib inline

Signal Processing is a main part of Data Driven Modeling cases

  • Usually observations are homogeneous
    • Pixels of image
    • Sequence of values in sound signal or any other time series
    • A Graphical representation of an artifact: City, building,...
  • Usually, we are looking for something on top of the observations (Categories)
    • Message of the sentence
    • A certain pattern in an image such as a face in the picture
  • All the methods are developed in a way to capture "invariances" within categories of interest
    • translation invariance
    • rotation invariance
    • scale invariance
    • Deformation
  • Hierarchical Representation and Compositionality
    • Text
    • Video
    • A Building
    • Cities

Fourier Series as Idealized Basis Functions

The Fourier series is linear algebra in infinite dimensions, where vectors are functions

In [2]:
#Base Filters 1D
def base_filter_1D(N):
    dd = -2*np.pi/N
    w = np.exp(dd*1j)
#     w = np.exp(-2*np.pi*1j/N)
    
    W = np.ones((N,N),dtype=complex)
    
    for i in range(N):
        for j in range(N):
            W[i,j] = np.power(w,i*j)

#     W = W/np.sqrt(N)
    return W

def DFT_1D_basis_vis(i):
    fig = plt.figure(figsize=(10,5))
    plt.subplot(1,2,1)
    #Real Part of the vector
    plt.plot(np.real(W)[i],'r')
    #Imaginary Part of the vector
    plt.title('real part')
    plt.axis('off')
    plt.subplot(1,2,2)
    plt.title('imaginary part')
    plt.plot(np.imag(W)[i],'b')
    plt.axis('off')
N = 256
W = base_filter_1D(N)
interact(DFT_1D_basis_vis,i=(0,N-1,1));

Now from Time domain to Frequency domain

In [3]:
# Two signals have the same frequencies but with a shift in time
N = 256
t = np.arange(N)
x1 = .4*np.sin(1*t+.1) + .6*np.cos(-15*t+.1) + .3*np.random.rand(N)
x2 = .4*np.sin(1*(t-2)+.1) + .6*np.cos(-15*(t-16)+.1) + .1*np.random.rand(N)
plt.plot(x1)
plt.plot(x2)
Out[3]:
[<matplotlib.lines.Line2D at 0x110be19d0>]
In [4]:
W = base_filter_1D(N)
X1 = W.dot(x1)
X2 = W.dot(x2)
In [5]:
plt.plot(np.abs(np.absolute(X1)),'b');
plt.plot(np.abs(np.absolute(X2)),'g');
In [6]:
N = 16
W1D = base_filter_1D(N)
def base_filter2d_vis(u=1,v=1):
    
    r = W1D[u][np.newaxis,:].T
    c = W1D[v][np.newaxis,:]
    W2 = r.dot(c)
    fig = plt.figure(figsize=(15,7))
    plt.subplot(1,2,1)
    plt.title('Real Part(CoSine Wave)')
    plt.imshow(np.real(W2),cmap=plt.cm.gray)
    plt.axis('off')
    plt.subplot(1,2,2)
    plt.title('Imaginary Part (Sine Wave)')
    plt.axis('off')
    plt.imshow(np.imag(W2),cmap=plt.cm.gray)
In [7]:
interact(base_filter2d_vis,u=(0,N-1,1),v=(0,N-1,1));
In [8]:
#Base Filters 2D
def base_filter_2D(N):
    W1D = base_filter_1D(N)*np.sqrt(N)
    W2D = np.ones((N,N,N,N),dtype=complex)
    
    for u in range(0,N):
        for v in range(0,N):
            r = W1D[u][np.newaxis,:].T
            c = W1D[v][np.newaxis,:]
            W2D[u,v,:,:] = r.dot(c)
    W2D = W2D/(np.sqrt(N*N))
    return W2D

Base Filters Dictionary

  • Note that these base filters are defined independent of the data!
In [47]:
N = 8
W2D = base_filter_2D(N)
print W2D.shape
fig = plt.figure(figsize=(7,7))
k =1 
for u in range(0,N):
    for v in range(0,N):
        W2 = W2D[u,v,:,:]
        plt.subplot(N,N,k)
        plt.imshow(np.real(W2),cmap=plt.cm.gray)
#         plt.imshow(np.imag(W2),cmap=plt.cm.gray)
        k = k +1
        plt.axis('off')
(8, 8, 8, 8)
In [50]:
# Example 1
# With shift stil the patters are similar in freq domain
N = 128
x = 5
y = 5
img = np.zeros((128,128))

for i in range(20):
    indx = np.random.randint(x,N-x)
    indy = np.random.randint(y,N-y)
    img[indx:indx+x,indy:indy+y] = 1
plt.subplot(2,2,1);
plt.imshow(img,cmap=plt.cm.gray);
plt.axis('off');

plt.subplot(2,2,2);
F_img = np.fft.fft2(img)

#To shift low pass (lower freq) to the center
F_img = np.fft.fftshift(F_img)
plt.imshow(np.log(np.absolute(F_img)));
plt.axis('off');


img = np.zeros((128,128))

x = 10
y = 10
for i in range(20):
    indx = np.random.randint(x,N-x)
    indy = np.random.randint(y,N-y)
    img[indx:indx+x,indy:indy+y] = 1
plt.subplot(2,2,3);
plt.imshow(img,cmap=plt.cm.gray);
plt.axis('off');


plt.subplot(2,2,4);
F_img = np.fft.fft2(img)

#To shift low pass (lower freq) to the center
F_img = np.fft.fftshift(F_img)
plt.imshow(np.log(np.absolute(F_img)));
plt.axis('off');

Conclusion of Fourier

No learning

Hard to learn hierarchies, scale, ...

And essentially not Data Driven

Conceptually, Fouries Transform is at the same level as Polynomial regression, where there is a parametric structure imposed to data set.


So a fundamental quesion has been always that wheter one can let the models learn the filters?

Until recently there were no alternative than desiging the filters, which means less capacity for the model


Representation Learning

Considering these factors?

  • translation invariance
  • rotation invariance
  • scale invariance
  • Deformation

  • Compositionality and Hierarchical Representation ## Convolutional Neural Networks (CNN) are seemingly a very good answer

Convolution

Continous case

Discrete case

1D Convolution

In [8]:
N = 100
t = np.linspace(0,4,num=N)
t =np.unique(np.concatenate((t,-t)))
r = t[-1]-t[0]
N = t.shape[0]
f = 1*np.sin(2.5*t) + 1*np.cos(1.5*t+.1)  + .63*np.random.rand(N)
f[f<0]=0
f = np.exp(-t)
f[t<0]=0

# f = np.ones(t.shape)*1
# c = 0
# f[t>=(c+1)]=0
# f[t<=(c-1)]=0


# sigma = 2
# f = np.exp(-(t-c)**2/2*sigma)


def vis_con(k=2):

    plt.subplot(2,1,1)
    plt.plot(t, f,'-b');
    
#     c = 0
    fgs = []
    sigma = 1
    
    for c in t:
        g = np.ones(t.shape)*1
        g[t>=(c+.5)]=0
        g[t<=(c-.5)]=0
        
#         g = np.exp(-((t-c)**2)/(2*(sigma**2)))
#         g = g/(np.sqrt(2*np.pi)*sigma)
        fgs.append(np.dot(f,g))
    
    
    
    
    g = np.ones(t.shape)*1
    g[t>=(k+.5)]=0
    g[t<=(k-.5)]=0
    
    
    #Guassian
#     g = np.exp(-((t-k)**2)/(2*(sigma**2)))
#     g = g/(np.sqrt(2*np.pi)*sigma)
#     g = 1/np.sqrt(2*np.pi*sigma**2)*np.exp(-((t-k)**2)/(2*sigma**2))
    
    fg = np.dot(f,g)
    plt.plot(t,g,'-r',linewidth=3)
    plt.ylabel('f')

    bottom1 = np.minimum(g, 0)
    bottom2 = np.minimum(0,f)
    bottom= np.minimum(bottom1,bottom2)
    top = np.minimum(g,f)
    plt.fill_between(t,top, bottom,facecolor='red',edgecolor="None", interpolate=True);
    plt.subplot(2,1,2);
    plt.plot(t,fgs)
#     plt.plot(t,np.convolve(f,g,mode='same'),'g');
    plt.plot(k,fg,'or');
    plt.xlim(t[0],t[-1])
    plt.ylabel('conv f*g')
    plt.xlabel('t')
interact(vis_con,k=(-4,4,.1));

It can be a signal smoothing way

Guassian Kernel

In [9]:
N = 100
t = np.linspace(0,4,num=N)
t =np.unique(np.concatenate((t,-t)))
r = t[-1]-t[0]
N = t.shape[0]
f = 1*np.sin(2.5*t) + 1*np.cos(1.5*t+.1)  + .63*np.random.rand(N)
f[f<0]=0
# f = np.exp(-t)
# f[t<0]=0

# f = np.ones(t.shape)*1
# c = 0
# f[t>=(c+1)]=0
# f[t<=(c-1)]=0


# sigma = 2
# f = np.exp(-(t-c)**2/2*sigma)


def vis_con(k=2,sigma=1):

    plt.subplot(2,1,1)
    plt.plot(t, f,'-b');
    
#     c = 0
    fgs = []
#     sigma = 1
    
    for c in t:
        g = np.ones(t.shape)*1
        g[t>=(c+.5)]=0
        g[t<=(c-.5)]=0
        
        g = np.exp(-((t-c)**2)/(2*(sigma**2)))
        g = g/(np.sqrt(2*np.pi)*sigma)
        fgs.append(np.dot(f,g))
    
    
    
    g = np.ones(t.shape)*1
    g[t>=(k+.5)]=0
    g[t<=(k-.5)]=0
    
    
    #Guassian
    g = np.exp(-((t-k)**2)/(2*(sigma**2)))
    g = g/(np.sqrt(2*np.pi)*sigma)
#     g = 1/np.sqrt(2*np.pi*sigma**2)*np.exp(-((t-k)**2)/(2*sigma**2))
    
    fg = np.dot(f,g)
    plt.plot(t,g,'-r',linewidth=3)
    plt.ylabel('f')

    bottom1 = np.minimum(np.abs(g), 0)
    bottom2 = np.minimum(0,np.abs(f))
    bottom= np.minimum(bottom1,bottom2)
    top = np.minimum(np.abs(g),np.abs(f))
    plt.fill_between(t,top, bottom,facecolor='red',edgecolor="None", interpolate=True);
    plt.subplot(2,1,2);
    plt.plot(t,fgs)
#     plt.plot(t,np.convolve(f,g,mode='same'),'g');
    plt.plot(k,fg,'or');
    plt.xlim(t[0],t[-1])
    plt.ylabel('conv f*g')
    plt.xlabel('t')
interact(vis_con,k=(-4,4,.1),sigma=(.01,10,.1));

What if the kernel is periodic or bigger than the domain of the data (f)?

It will be simply an average over f

In [54]:
N = 100
t = np.linspace(0,4,num=N)
t =np.unique(np.concatenate((t,-t)))
r = t[-1]-t[0]
N = t.shape[0]
f = 1*np.sin(2.5*t) + 1*np.cos(1.5*t+.1)  + .63*np.random.rand(N)
# f[f<0]=0
# f = np.exp(-t)
# f[t<0]=0

# f = np.ones(t.shape)*1
# c = 0
# f[t>=(c+1)]=0
# f[t<=(c-1)]=0


sigma = 2
# f = np.exp(-(t-c)**2/2*sigma)


def vis_con(k=2,sigma=1):

    plt.subplot(2,1,1)
    plt.plot(t, f,'-b');
    
#     c = 0
    fgs = []
#     sigma = 1
    
    for c in t:
        g = np.ones(t.shape)*1
        g[t>=(c+sigma)]=0
        g[t<=(c-sigma)]=0
        
#         g = np.exp(-((t-c)**2)/(2*(sigma**2)))
#         g = g/(np.sqrt(2*np.pi)*sigma)
        fgs.append(np.dot(f,g))
    
    
    
    g = np.ones(t.shape)*1
    g[t>=(k+sigma)]=0
    g[t<=(k-sigma)]=0
    
    
    #Guassian
#     g = np.exp(-((t-k)**2)/(2*(sigma**2)))
#     g = g/(np.sqrt(2*np.pi)*sigma)
#     g = 1/np.sqrt(2*np.pi*sigma**2)*np.exp(-((t-k)**2)/(2*sigma**2))
    
    fg = np.dot(f,g)
    plt.plot(t,g,'-r',linewidth=3)
    plt.ylabel('f')

    bottom1 = np.minimum(np.abs(g), 0)
    bottom2 = np.minimum(0,np.abs(f))
    bottom= np.minimum(bottom1,bottom2)
    top = np.minimum(np.abs(g),np.abs(f))
    plt.fill_between(t,top, bottom,facecolor='red',edgecolor="None", interpolate=True);
    plt.subplot(2,1,2);
    plt.plot(t,fgs)
#     plt.plot(t,np.convolve(f,g,mode='same'),'g');
    plt.plot(k,fg,'or');
    plt.xlim(t[0],t[-1])
    plt.ylabel('conv f*g')
    plt.xlabel('t')
interact(vis_con,k=(-4,4,.1),sigma=(.01,10,.1));