\[ \min_{\beta \in \mathbb{R}} -\left[ \frac{1}{N} \sum_{i=1}^N y_i X_{i:}\beta - log(1 + e^{X_{i:}\beta}) \right] + \lambda\left[ \frac{1}{2}(1 - \alpha) \|\beta\|_2^2 + \alpha\|\beta\|_1 \right] \]

model_logit <- train(modelFormula,
                     data=data,
                     method="glmnet",
                     preProcess=c("scale", "center"),
                     trControl=controls)

\[ \hat{p}_{mk} = \frac{1}{N_m} \sum_{x_i \in R_m} I(y_i = k) \]

model_tree <- train(modelFormula,
                     data=data,
                     method="rpart",
                     preProcess=c("scale", "center"),
                     trControl=controls)

\[ \hat{C}_{rf}^B = \text{majority vote} \{\hat{C}_b(x)\}_1^B \]

model_forest <- train(modelFormula,
                     data=data,
                     method="rf",
                     preProcess=c("scale", "center"),
                     trControl=controls)

\[ \hat{y}_i^t = \sum_{k=1}^tf_k(x_i) = \hat{y}_i^{(t-1)} + f_t(x_i) \]

model_xgboost <- train(modelFormula,
                     data=data,
                     method="xgbTree",
                     preProcess=c("scale", "center"),
                     trControl=controls)

model_boostforest <- train(modelFormula,
                       data=data,
                       method="xgbTree",
                       preProcess=c("scale", "center"),
                       trControl=controls, 
                       num_parallel_tree=10)

model_nnet <- train(modelFormula,
                     data=data,
                     method="nnet",
                     preProcess=c("scale", "center"),
                     trControl=controls, trace=FALSE)

x_train <- build.x(modelFormula, data=data,
                   contrasts=FALSE, sparse=FALSE)
y_train <- build.y(modelFormula, data=data) %>% as.integer() - 1

x_val <- build.x(modelFormula, data=validate,
                   contrasts=FALSE, sparse=FALSE)
y_val <- build.y(modelFormula, data=validate) %>% as.integer() - 1

net_mxnet <- mx.symbol.Variable('data') %>%
    # drop out 20% of predictors
    mx.symbol.Dropout(p=0.2, name='Predictor_Dropout') %>%
    # a fully connected layer with 256 units
    mx.symbol.FullyConnected(num_hidden=256, name='fc_1') %>%
    # use the rectified linear unit (relu) for the activation function
    mx.symbol.Activation(act_type='relu', name='relu_1') %>%
    # drop out 50% of the units
    mx.symbol.Dropout(p=0.5, name='dropout_1') %>%
    # a fully connected layer with 128 units
    mx.symbol.FullyConnected(num_hidden=128, name='fc_2') %>%
    # use the rectified linear unit (relu) for the activation function
    mx.symbol.Activation(act_type='relu', name='relu_2') %>%
    # drop out 50% of the units
    mx.symbol.Dropout(p=0.5, name='dropout_2') %>%
    # fully connect to the output layer which has just the 1 unit
    mx.symbol.FullyConnected(num_hidden=1, name='out') %>%
    # use the sigmoid output
    mx.symbol.LogisticRegressionOutput(name='output')

graph.viz(net_mxnet, direction='LR')

# use four CPU threads
Sys.setenv('MXNET_CPU_WORKER_NTHREADS'=4)

# set the random seed
mx.set.seed(1234)

\[ \text{logloss} = -\left( y*\text{log}(p) + (1-y)*\text{log}(1-p) \right) \]

# custom log-loss function
mx.metric.logloss <- mx.metric.custom("logloss", function(label, pred){
    return(Metrics::logLoss(label, pred))
})

# validation logger
logger <- mx.metric.logger$new()

# train the model
mod_mxnet <- mx.model.FeedForward.create(
    symbol            = net_mxnet,    # the symbolic network
    X                 = x_train, # the predictors
    y                 = y_train, # the response
    optimizer         = "adam", # using the Adam optimization method
    eval.data         = list(data=x_val, label=y_val), # validation data
    ctx               = mx.cpu(), # use the cpu for training
    eval.metric       = mx.metric.logloss, # evaluate with log-loss
    num.round         = 50,     # 50 epochs
    learning.rate     = 0.001,   # learning rate
    array.batch.size  = 128,    # batch size
    array.layout      = "rowmajor",  # the data is stored in row major format
    epoch.end.callback= mx.callback.log.train.metric(1, logger),
    verbose           = FALSE
)

library(keras)

# CPU
install_keras()

# GPU
install_keras(tensorflow='GPU')

net_keras <- keras_model_sequential() %>% 
    # fully connected layer
    layer_dense(units = 512, activation = 'relu', 
                input_shape=dim(x_train)[[-1]], name='fc1') %>% 
    # batch normalization
    layer_batch_normalization(name='batchnorm1') %>% 
    # dropout
    layer_dropout(rate=0.5, name='dropout1') %>% 
    # fully connected layer
    layer_dense(units=256, activation='relu', name='fc2') %>% 
    # batch normalization
    layer_batch_normalization(name='batchnorm2') %>% 
    # dropout
    layer_dropout(rate=0.5, name='dropout2') %>% 
    # output layer
    layer_dense(units=1, activation="sigmoid", name='out')

net_keras %>% compile(
    optimizer = "adam",
    loss = 'binary_crossentropy',
    metrics = c("accuracy")
)

history <- net_keras %>% 
    fit(x=x_train, y=y_train,
        epochs=10,
        batch_size=128,
        validation_data=list(x_val, y_val),
        verbose=FALSE, view_metrics=TRUE,
        callbacks=list(
            callback_early_stopping(monitor='val_acc',
                                    patience=5),
            callback_model_checkpoint(filepath='model_deep.h5',
                                      monitor='val_loss',
                                      save_best_only=TRUE),
            callback_reduce_lr_on_plateau(monitor='val_loss',
                                          factor=0.1,
                                          patience=3),
            callback_tensorboard("logs/run_a")
        )
    )

net_cnn <- keras_model_sequential() %>% 
    layer_conv_2d(filters=32, kernel_size=c(3, 3), activation="relu",
                  input_shape=c(28, 28, 1)) %>% 
    layer_max_pooling_2d(pool_size=c(2, 2)) %>% 
    layer_conv_2d(filters=64, kernel_size=c(3, 3), activation="relu") %>% 
    layer_max_pooling_2d(pool_size=c(2, 2)) %>% 
    layer_conv_2d(filters=64, kernel_size=c(3, 3), activation="relu") %>% 
    layer_flatten() %>% 
    layer_dense(units=64, activation="relu") %>% 
    layer_dense(units=10, activation="softmax")

net_cnn %>% 
    compile(
        optimizer = "rmsprop",
        loss = "categorical_crossentropy",
        metrics = c("accuracy")
    )

history_cnn <- net_cnn %>% 
    fit(
        x=train_images, y=train_labels, 
        validation_split=0.2,
        epochs=5, batch_size=64,
        view_metrics=TRUE, verbose=FALSE,
        callbacks=list(
            callback_early_stopping(monitor='val_acc',
                                    patience=3),
            callback_model_checkpoint(filepath='mnist_conv.h5',
                                      monitor='val_loss',
                                      save_best_only=TRUE),
            callback_reduce_lr_on_plateau(monitor='val_loss',
                                          factor=0.1,
                                          patience=2),
            callback_tensorboard("logs/run_b")
        )
    )