# Code source: Jaques Grobler
# License: BSD 3 clause
import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.linear_model import SGDRegressor
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_wine
from sklearn.model_selection import train_test_split

## Iris dataset example
wine = load_wine()
x, y = wine.data, wine.target

X_train, X_test, y_train, y_test=train_test_split(x, y, test_size=0.15)

# Create linear regression object without regularization
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(X_train, y_train)

# Make predictions using the testing set
prediction1 = regr.predict(X_test)

# The mean squared error
print("Mean squared error - without regularization: %.6f" % mean_squared_error(y_test, prediction1))

# Create linear regression object with L2 regularization - Ridge (Linear Regresion model)
regr2 = linear_model.Ridge(alpha = 0.01, solver ='lsqr', tol = 1e-3)

# Train the model using the training sets
regr2.fit(X_train, y_train)

# Make predictions using the testing set
prediction2 = regr2.predict(X_test)

# The mean squared error
print("Mean squared error - L2 regularization: %.6f" % mean_squared_error(y_test, prediction2))

# Create linear regression object with L1 regularization - Lasso (Linear Regresion model)
regr3 = linear_model.Lasso(alpha = 0.01, tol = 1e-3)

# Train the model using the training sets
regr3.fit(X_train, y_train)

# Make predictions using the testing set
prediction3 = regr3.predict(X_test)

# The mean squared error
print("Mean squared error - L1 regularization: %.6f" % mean_squared_error(y_test, prediction3))

# Create linear regression object with L2 regularization - Ridge using the SGDRegressor
regr4 = make_pipeline(StandardScaler(), SGDRegressor(max_iter = 100, alpha = 0.01, tol = 1e-3))

# Train the model using the training sets
regr4.fit(X_train, y_train)

prediction4 = regr4.predict(X_test)

# The mean squared error
print("Mean squared error - SGD-L2: %.6f" % mean_squared_error(y_test, prediction4))

# Create linear regression object with L1 regularization - Lasso using the SGDRegressor
regr5 = make_pipeline(StandardScaler(), SGDRegressor(max_iter = 100, penalty = 'l1', alpha = 0.01, tol = 1e-3))

# Train the model using the training sets
regr5.fit(X_train, y_train)

# Make predictions using the testing set
prediction5 = regr5.predict(X_test)

# The mean squared error
print("Mean squared error - SGD-L1: %.6f" % mean_squared_error(y_test, prediction5))
