# python_neural_net.py  NEURAL NET April 2024 based on # ass2s24q5_revised.py 
  
# A. Colin Cameron, Dept. of Economics, University of California - Davis

# This neural net example comes from Aurelion Gerien "Hands-on Machine Learning
# with Scikit-Learn, Keras and TensorFlow" 3rd edition chapter 10
# https://github.com/ageron/handson-ml3/

# This uses Keras - for documentation see https://keras.io/api/

# The program uses tensorflow which is not in the base environment for Anaconda
# And it uses Keras which is loaded within tensorflow
# Make a special environment for this program (do not use the base enviroment)
# This environment needs the modules
#   tensorflow
#   scikitlearn
#   os 

# Set up the working directory 
import os
os.getcwd()
os.chdir("c:/Users/ccameron/Dropbox/Desktop/Teaching/240f/assignments/")
os.getcwd()

# Import tensorflow which also brings in tensorflow.keras
import tensorflow as tf

# Read in the data which are provided in sklearn
# https://scikit-learn.org/stable/datasets/real_world.html
from sklearn.datasets import fetch_california_housing
housing = fetch_california_housing()
# FInd information about data and its organization
print(housing)
# Shows that housing.data has the features and housing.target has the outcome
print(housing.feature_names)
print(housing.target_names)
print(housing.data.shape, housing.target.shape)
# Mean of data - housing price in units of $100,000
import numpy as np
np.average(housing.target,axis=0)
np.average(housing.data,axis=0)

# Load and split the California housing dataset
from sklearn.model_selection import train_test_split
# First split into analysis data (75%) and test data (25%) 
X_train_full, X_test, y_train_full, y_test = train_test_split(
    housing.data, housing.target, train_size=0.75, random_state=42)
print(X_train_full.shape, y_train_full.shape,X_test.shape,y_test.shape)
# Second split analysis data into training data (75%) and validation data (25%)
X_train, X_valid, y_train, y_valid = train_test_split(
    X_train_full, y_train_full, train_size=0.75, random_state=42)
print(X_train.shape, y_train.shape,X_valid.shape, y_valid.shape)

# IMPORTANT: Clear the session to reset the name counters
# This is in case you have already run the code below
tf.keras.backend.clear_session()

# IMPORTANT: Set the seed for reproducability 
# It is not enough to e.g. np.random.seed(42) and tf.random.set_seed(42)
# Instead do the following more complicated from 
# https://stackoverflow.com/questions/36288235/how-to-get-stable-results-with-tensorflow-setting-random-seed

import random
SEED = 0
def set_seeds(seed=SEED):
    os.environ['PYTHONHASHSEED'] = str(seed)
    random.seed(seed)
    tf.random.set_seed(seed)
    np.random.seed(seed)
def set_global_determinism(seed=SEED):
    set_seeds(seed=seed)
    os.environ['TF_DETERMINISTIC_OPS'] = '1'
    os.environ['TF_CUDNN_DETERMINISTIC'] = '1'   
set_global_determinism(seed=SEED)

# (1) Begin analysis - first define the neural net

# Normalize inputs to have mean 0 and variance 1
norm_layer = tf.keras.layers.Normalization(input_shape=X_train.shape[1:])

# Use a Sequential neural net
# The simplest neural net with a single stack of layers connected sequentially
# For nonsequential uses the Functional API https://keras.io/api/models/
 
# Here there are 3 hidden layers 
# And the final command says not to transform the output (no activationh)
# (We would transform if e.g. want to be between 0 and 1)
# The activation function for the hidden layers is reclu 
# ReLU Recitifed linear unit activation function (: max(x,0))
# For other activation functions see https://keras.io/api/layers/activations/
model = tf.keras.Sequential([
    norm_layer,
    tf.keras.layers.Dense(50, activation="relu"),
    tf.keras.layers.Dense(50, activation="relu"),
    tf.keras.layers.Dense(50, activation="relu"),
    tf.keras.layers.Dense(1)
])

# Summarize model and parameters
# Weight parameters are slope coefficients and bias parameters are intercepts
# Here we have three hidden layers with each having 50 units
# The 8 inputs give 50*8 weights + 50 biases = 450 parameters
# Then 50 inputs give 50*50 weights + 50 biases = 2550 parameters
# Then 50 inputs give 50*50 weights + 50 biases = 2550 parameters
# Then 50 inputs to get output give 50 weights + 1 bias 
model.summary()

# Model's list of layers
model.layers

# Get the weights and biases for the first hidden layer
# These are the initialized values - 0 for biases and random for weights
hidden1 = model.layers[1]
hidden1.name
model.get_layer('dense') is hidden1
weights, biases = hidden1.get_weights()
weights.shape
weights
biases.shape
biases

# (2) Define the optimization method, loss function, ...

# Optimzers are given at https://keras.io/api/optimizers/ 
# Here we use the Adam optimizer which as a stochastic gradient descent method
# The specified learning rate of 0.001 is the default for Adam
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)

# Configure the model using the compile method
# For other methods see https://keras.io/api/models/model_training_apis/
# Loss is MSE See https://keras.io/api/metrics/ for other loss functions
model.compile(loss="mse", optimizer=optimizer, metrics=["RootMeanSquaredError"])

# (3) Run the neural net

# Normalize inputs to mean 0 and variance 1
norm_layer.adapt(X_train)

# Train the model
# Here only 50 epochs or iterations
history = model.fit(X_train, y_train, batch_size=32, epochs=30,
                    validation_data=(X_valid, y_valid))

# Details on the iterations
history.params
print(history.epoch)
print(history.history)

# (4) Evaluate how well the neural net performs in the test data set
mse_test, rmse_test = model.evaluate(X_test, y_test)
rmse_test

# Predict for the first three observations in the test dataset
X_new = X_test[:3]
X_new
y_pred = model.predict(X_new)
y_pred
y_new = y_test[:3]
y_new

# END