# Code source: Jaques Grobler
# License: BSD 3 clause
import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.linear_model import SGDRegressor
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

# Load the diabetes dataset
diabetes_X, diabetes_y = datasets.load_diabetes(return_X_y=True)

# Use only one feature
diabetes_X = diabetes_X[:, np.newaxis, 2]

# Split the data into training/testing sets
diabetes_X_train = diabetes_X[:-20]
diabetes_X_test = diabetes_X[-20:]

# Split the targets into training/testing sets
diabetes_y_train = diabetes_y[:-20]
diabetes_y_test = diabetes_y[-20:]

# Create linear regression object without regularization
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr.predict(diabetes_X_test)

# Plot outputs
plt.scatter(diabetes_X_test, diabetes_y_test, color="black")
plt.plot(diabetes_X_test, diabetes_y_pred, color="blue", linewidth=1.5, label ='Without reg.')

# Create linear regression object with L2 normalization - Ridge
regr2 = linear_model.Ridge(alpha = 0.15, solver ='lsqr')

# Train the model using the training sets
regr2.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred2 = regr2.predict(diabetes_X_test)

# Plot outputs
plt.plot(diabetes_X_test, diabetes_y_pred2, color="red", linewidth=1.5, label ='Ridge - L2')

# Create linear regression object with L1 regularization - Lasso
regr3 = linear_model.Lasso(alpha = 0.15)

# Train the model using the training sets
regr3.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred3 = regr3.predict(diabetes_X_test)

# Plot outputs
plt.plot(diabetes_X_test, diabetes_y_pred3, color="green", linewidth=1.5, label ='Lasso - L1')

# Create linear regression object - SGDRegressor with L2 regularization
reg4 = make_pipeline(StandardScaler(), SGDRegressor(max_iter = 1000, alpha = 0.15, tol = 1e-5))

# Train the model using the training sets
reg4.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred4 = reg4.predict(diabetes_X_test)

# Plot outputs
plt.plot(diabetes_X_test, diabetes_y_pred4, color="cyan", linewidth=1.5, label ='SGD Regressor - L2')

# Create linear regression object - SGDRegressor with L1 regularization
reg5 = make_pipeline(StandardScaler(), SGDRegressor(max_iter = 1000, penalty = 'l1', alpha = 0.15, tol = 1e-5))

# Train the model using the training sets
reg5.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred5 = reg5.predict(diabetes_X_test)

# Plot outputs
plt.plot(diabetes_X_test, diabetes_y_pred5, color="pink", linewidth=1.5, label ='SGD Regressor - L1')

plt.xticks(())
plt.yticks(())
plt.legend()
plt.title('λ = 0.15')
plt.show()
