Unsupervised Neural network Models

Restricted Boltzmann Machine (RBM)

  • Use input data as visible vector, this hidden vector is a transformed version of the input data
  • The hidden vector represents the extract underlying features
  • RBMs are not normally used currently, GAN and VAE are often used
In [1]:
from sklearn import linear_model, datasets, metrics
from sklearn.model_selection import train_test_split
from sklearn.neural_network import BernoulliRBM
In [14]:
X, y = datasets.load_digits(return_X_y=True)
X = (X - np.min(X, 0)) / (np.max(X, 0) + 0.0001)  # 0-1 scaling
In [15]:
import numpy as np
X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.2, random_state=0)
In [41]:
rbm = BernoulliRBM(random_state=0, verbose=True)
rbm.learning_rate = 0.06
rbm.n_iter = 100
rbm.n_components = 100

rbm.fit(X_train)

[BernoulliRBM] Iteration 1, pseudo-likelihood = -27.63, time = 0.05s
[BernoulliRBM] Iteration 2, pseudo-likelihood = -27.03, time = 0.05s
[BernoulliRBM] Iteration 3, pseudo-likelihood = -25.98, time = 0.04s
[BernoulliRBM] Iteration 4, pseudo-likelihood = -25.09, time = 0.04s
[BernoulliRBM] Iteration 5, pseudo-likelihood = -23.97, time = 0.04s
[BernoulliRBM] Iteration 6, pseudo-likelihood = -23.77, time = 0.04s
[BernoulliRBM] Iteration 7, pseudo-likelihood = -23.28, time = 0.04s
[BernoulliRBM] Iteration 8, pseudo-likelihood = -22.70, time = 0.04s
[BernoulliRBM] Iteration 9, pseudo-likelihood = -22.41, time = 0.04s
[BernoulliRBM] Iteration 10, pseudo-likelihood = -22.41, time = 0.04s
[BernoulliRBM] Iteration 11, pseudo-likelihood = -22.87, time = 0.04s
[BernoulliRBM] Iteration 12, pseudo-likelihood = -22.62, time = 0.04s
[BernoulliRBM] Iteration 13, pseudo-likelihood = -21.98, time = 0.04s
[BernoulliRBM] Iteration 14, pseudo-likelihood = -22.17, time = 0.04s
[BernoulliRBM] Iteration 15, pseudo-likelihood = -22.12, time = 0.04s
[BernoulliRBM] Iteration 16, pseudo-likelihood = -21.57, time = 0.04s
[BernoulliRBM] Iteration 17, pseudo-likelihood = -21.45, time = 0.04s
[BernoulliRBM] Iteration 18, pseudo-likelihood = -21.42, time = 0.04s
[BernoulliRBM] Iteration 19, pseudo-likelihood = -21.28, time = 0.04s
[BernoulliRBM] Iteration 20, pseudo-likelihood = -20.98, time = 0.04s
[BernoulliRBM] Iteration 21, pseudo-likelihood = -20.85, time = 0.04s
[BernoulliRBM] Iteration 22, pseudo-likelihood = -20.62, time = 0.04s
[BernoulliRBM] Iteration 23, pseudo-likelihood = -20.76, time = 0.03s
[BernoulliRBM] Iteration 24, pseudo-likelihood = -20.65, time = 0.03s
[BernoulliRBM] Iteration 25, pseudo-likelihood = -20.26, time = 0.04s
[BernoulliRBM] Iteration 26, pseudo-likelihood = -20.31, time = 0.05s
[BernoulliRBM] Iteration 27, pseudo-likelihood = -20.16, time = 0.05s
[BernoulliRBM] Iteration 28, pseudo-likelihood = -20.23, time = 0.04s
[BernoulliRBM] Iteration 29, pseudo-likelihood = -20.24, time = 0.05s
[BernoulliRBM] Iteration 30, pseudo-likelihood = -20.09, time = 0.04s
[BernoulliRBM] Iteration 31, pseudo-likelihood = -19.76, time = 0.03s
[BernoulliRBM] Iteration 32, pseudo-likelihood = -20.02, time = 0.03s
[BernoulliRBM] Iteration 33, pseudo-likelihood = -19.74, time = 0.03s
[BernoulliRBM] Iteration 34, pseudo-likelihood = -19.53, time = 0.03s
[BernoulliRBM] Iteration 35, pseudo-likelihood = -19.72, time = 0.03s
[BernoulliRBM] Iteration 36, pseudo-likelihood = -19.73, time = 0.04s
[BernoulliRBM] Iteration 37, pseudo-likelihood = -19.47, time = 0.04s
[BernoulliRBM] Iteration 38, pseudo-likelihood = -19.45, time = 0.04s
[BernoulliRBM] Iteration 39, pseudo-likelihood = -19.25, time = 0.04s
[BernoulliRBM] Iteration 40, pseudo-likelihood = -19.29, time = 0.04s
[BernoulliRBM] Iteration 41, pseudo-likelihood = -19.25, time = 0.04s
[BernoulliRBM] Iteration 42, pseudo-likelihood = -19.32, time = 0.04s
[BernoulliRBM] Iteration 43, pseudo-likelihood = -19.20, time = 0.04s
[BernoulliRBM] Iteration 44, pseudo-likelihood = -19.27, time = 0.03s
[BernoulliRBM] Iteration 45, pseudo-likelihood = -19.18, time = 0.04s
[BernoulliRBM] Iteration 46, pseudo-likelihood = -19.12, time = 0.04s
[BernoulliRBM] Iteration 47, pseudo-likelihood = -19.00, time = 0.03s
[BernoulliRBM] Iteration 48, pseudo-likelihood = -19.15, time = 0.03s
[BernoulliRBM] Iteration 49, pseudo-likelihood = -19.35, time = 0.04s
[BernoulliRBM] Iteration 50, pseudo-likelihood = -18.99, time = 0.03s
[BernoulliRBM] Iteration 51, pseudo-likelihood = -19.03, time = 0.03s
[BernoulliRBM] Iteration 52, pseudo-likelihood = -18.75, time = 0.03s
[BernoulliRBM] Iteration 53, pseudo-likelihood = -19.11, time = 0.04s
[BernoulliRBM] Iteration 54, pseudo-likelihood = -19.00, time = 0.04s
[BernoulliRBM] Iteration 55, pseudo-likelihood = -18.85, time = 0.04s
[BernoulliRBM] Iteration 56, pseudo-likelihood = -18.66, time = 0.03s
[BernoulliRBM] Iteration 57, pseudo-likelihood = -18.90, time = 0.04s
[BernoulliRBM] Iteration 58, pseudo-likelihood = -18.67, time = 0.04s
[BernoulliRBM] Iteration 59, pseudo-likelihood = -19.04, time = 0.04s
[BernoulliRBM] Iteration 60, pseudo-likelihood = -19.02, time = 0.04s
[BernoulliRBM] Iteration 61, pseudo-likelihood = -18.76, time = 0.04s
[BernoulliRBM] Iteration 62, pseudo-likelihood = -18.89, time = 0.03s
[BernoulliRBM] Iteration 63, pseudo-likelihood = -18.79, time = 0.03s
[BernoulliRBM] Iteration 64, pseudo-likelihood = -18.78, time = 0.03s
[BernoulliRBM] Iteration 65, pseudo-likelihood = -18.60, time = 0.03s
[BernoulliRBM] Iteration 66, pseudo-likelihood = -18.73, time = 0.03s
[BernoulliRBM] Iteration 67, pseudo-likelihood = -18.79, time = 0.04s
[BernoulliRBM] Iteration 68, pseudo-likelihood = -18.67, time = 0.03s
[BernoulliRBM] Iteration 69, pseudo-likelihood = -18.80, time = 0.03s
[BernoulliRBM] Iteration 70, pseudo-likelihood = -18.53, time = 0.03s
[BernoulliRBM] Iteration 71, pseudo-likelihood = -18.71, time = 0.03s
[BernoulliRBM] Iteration 72, pseudo-likelihood = -18.54, time = 0.03s
[BernoulliRBM] Iteration 73, pseudo-likelihood = -18.69, time = 0.04s
[BernoulliRBM] Iteration 74, pseudo-likelihood = -18.57, time = 0.03s
[BernoulliRBM] Iteration 75, pseudo-likelihood = -18.55, time = 0.04s
[BernoulliRBM] Iteration 76, pseudo-likelihood = -18.67, time = 0.04s
[BernoulliRBM] Iteration 77, pseudo-likelihood = -18.87, time = 0.04s
[BernoulliRBM] Iteration 78, pseudo-likelihood = -18.58, time = 0.04s
[BernoulliRBM] Iteration 79, pseudo-likelihood = -18.61, time = 0.04s
[BernoulliRBM] Iteration 80, pseudo-likelihood = -18.43, time = 0.04s
[BernoulliRBM] Iteration 81, pseudo-likelihood = -18.49, time = 0.04s
[BernoulliRBM] Iteration 82, pseudo-likelihood = -18.55, time = 0.04s
[BernoulliRBM] Iteration 83, pseudo-likelihood = -18.66, time = 0.04s
[BernoulliRBM] Iteration 84, pseudo-likelihood = -18.59, time = 0.03s
[BernoulliRBM] Iteration 85, pseudo-likelihood = -18.54, time = 0.04s
[BernoulliRBM] Iteration 86, pseudo-likelihood = -18.34, time = 0.04s
[BernoulliRBM] Iteration 87, pseudo-likelihood = -18.45, time = 0.03s
[BernoulliRBM] Iteration 88, pseudo-likelihood = -18.42, time = 0.03s
[BernoulliRBM] Iteration 89, pseudo-likelihood = -18.39, time = 0.03s
[BernoulliRBM] Iteration 90, pseudo-likelihood = -18.50, time = 0.03s
[BernoulliRBM] Iteration 91, pseudo-likelihood = -18.37, time = 0.04s
[BernoulliRBM] Iteration 92, pseudo-likelihood = -18.54, time = 0.03s
[BernoulliRBM] Iteration 93, pseudo-likelihood = -18.51, time = 0.03s
[BernoulliRBM] Iteration 94, pseudo-likelihood = -18.45, time = 0.03s
[BernoulliRBM] Iteration 95, pseudo-likelihood = -18.40, time = 0.03s
[BernoulliRBM] Iteration 96, pseudo-likelihood = -18.55, time = 0.03s
[BernoulliRBM] Iteration 97, pseudo-likelihood = -18.37, time = 0.04s
[BernoulliRBM] Iteration 98, pseudo-likelihood = -18.43, time = 0.04s
[BernoulliRBM] Iteration 99, pseudo-likelihood = -18.37, time = 0.04s
[BernoulliRBM] Iteration 100, pseudo-likelihood = -18.70, time = 0.03s
Out[41]:
BernoulliRBM(batch_size=10, learning_rate=0.06, n_components=100, n_iter=100,
             random_state=0, verbose=True)
In [42]:
# each node has a bias
rbm.intercept_hidden_.shape # 100, biases of the hidden units
rbm.intercept_visible_.shape # 64, biases of the visible units
Out[42]:
(64,)
In [43]:
rbm_train = rbm.transform(X_train)
In [44]:
logistic = linear_model.LogisticRegression(solver='newton-cg', tol=1)
logistic.C = 6000

logistic.fit(rbm_train, Y_train)
Out[44]:
LogisticRegression(C=6000, class_weight=None, dual=False, fit_intercept=True,
                   intercept_scaling=1, l1_ratio=None, max_iter=100,
                   multi_class='auto', n_jobs=None, penalty='l2',
                   random_state=None, solver='newton-cg', tol=1, verbose=0,
                   warm_start=False)
In [45]:
y_pred = logistic.predict(rbm.transform(X_test))
In [49]:
from sklearn.metrics import confusion_matrix
cf_matrix = confusion_matrix(Y_test, y_pred)
cf_matrix
Out[49]:
array([[27,  0,  0,  0,  0,  0,  0,  0,  0,  0],
       [ 0, 34,  0,  0,  0,  0,  0,  0,  0,  1],
       [ 0,  0, 34,  1,  0,  0,  0,  0,  1,  0],
       [ 0,  0,  0, 28,  0,  0,  0,  0,  1,  0],
       [ 0,  0,  0,  0, 29,  0,  0,  1,  0,  0],
       [ 0,  0,  0,  0,  0, 39,  0,  0,  0,  1],
       [ 0,  1,  0,  0,  0,  0, 43,  0,  0,  0],
       [ 0,  0,  0,  0,  0,  0,  0, 39,  0,  0],
       [ 0,  2,  0,  1,  1,  0,  0,  1, 34,  0],
       [ 0,  0,  0,  3,  0,  1,  0,  0,  0, 37]])