Linear Regression: Interactive Predictive Modeling Tutorial

Description

Linear Regression is a fundamental machine learning algorithm used for predicting a continuous output variable based on one or more input features. It works by finding the best-fitting linear relationship between the input and output variables.

How It Works

Model Representation:
- y = mx + b (Simple Linear Regression)
- y = β₀ + β₁x₁ + β₂x₂ + ... + βₙxₙ (Multiple Linear Regression)
- where y is the predicted value, x are features, and β are coefficients
Cost Function:
- Mean Squared Error (MSE): J(θ) = (1/2m)Σ(hᵢ - yᵢ)²
- where h is the prediction and y is the actual value
Gradient Descent:
- Update coefficients: θⱼ = θⱼ - α∂J/∂θⱼ
- where α is the learning rate

Complexity Analysis

Time Complexity

Training: O(n × m × i)
- n: number of features
- m: number of training examples
- i: number of iterations
Prediction: O(n)
- Linear time with respect to number of features

Space Complexity

Model Storage: O(n)
- Coefficients: O(n)
- Intermediate calculations: O(m)

Visualization

Learning Rate:

Click 'Generate Data' to begin

Implementation


import numpy as np

class LinearRegression:
    def __init__(self, learning_rate=0.01, n_iterations=1000):
        self.learning_rate = learning_rate
        self.n_iterations = n_iterations
        self.weights = None
        self.bias = None
        self.history = {'loss': []}

    def fit(self, X, y):
        # Initialize parameters
        n_samples, n_features = X.shape
        self.weights = np.zeros(n_features)
        self.bias = 0
        
        # Gradient descent
        for _ in range(self.n_iterations):
            # Forward pass
            y_predicted = self._predict(X)
            
            # Calculate gradients
            dw = (1/n_samples) * np.dot(X.T, (y_predicted - y))
            db = (1/n_samples) * np.sum(y_predicted - y)
            
            # Update parameters
            self.weights -= self.learning_rate * dw
            self.bias -= self.learning_rate * db
            
            # Calculate and store loss
            loss = self._mse(y, y_predicted)
            self.history['loss'].append(loss)

    def predict(self, X):
        return self._predict(X)

    def _predict(self, X):
        return np.dot(X, self.weights) + self.bias

    def _mse(self, y_true, y_pred):
        return np.mean((y_true - y_pred) ** 2)

# Example usage
if __name__ == "__main__":
    # Generate sample data
    np.random.seed(0)
    X = 2 * np.random.rand(100, 1)
    y = 4 + 3 * X + np.random.randn(100, 1)

    # Create and train model
    model = LinearRegression(learning_rate=0.01, n_iterations=1000)
    model.fit(X, y)

    # Make predictions
    X_test = np.array([[0], [2]])
    y_pred = model.predict(X_test)

    print("Weights:", model.weights)
    print("Bias:", model.bias)
    print("Predictions:", y_pred)


#include <vector>
#include <iostream>
#include <cmath>
#include <random>

class LinearRegression {
private:
    double learning_rate;
    int n_iterations;
    std::vector<double> weights;
    double bias;

    double predict(const std::vector<double>& x) {
        double y_pred = bias;
        for (size_t i = 0; i < x.size(); ++i) {
            y_pred += weights[i] * x[i];
        }
        return y_pred;
    }

public:
    LinearRegression(double lr = 0.01, int iterations = 1000) 
        : learning_rate(lr), n_iterations(iterations) {}

    void fit(const std::vector<std::vector<double>>& X, 
            const std::vector<double>& y) {
        int n_samples = X.size();
        int n_features = X[0].size();

        // Initialize parameters
        weights = std::vector<double>(n_features, 0.0);
        bias = 0.0;

        // Gradient descent
        for (int iter = 0; iter < n_iterations; ++iter) {
            double dw_sum = 0.0;
            std::vector<double> dw(n_features, 0.0);
            double db = 0.0;

            // Calculate gradients
            for (int i = 0; i < n_samples; ++i) {
                double y_pred = predict(X[i]);
                double error = y_pred - y[i];

                for (int j = 0; j < n_features; ++j) {
                    dw[j] += error * X[i][j];
                }
                db += error;
            }

            // Update parameters
            for (int j = 0; j < n_features; ++j) {
                weights[j] -= learning_rate * dw[j] / n_samples;
            }
            bias -= learning_rate * db / n_samples;
        }
    }

    std::vector<double> predict(const std::vector<std::vector<double>>& X) {
        std::vector<double> predictions;
        for (const auto& x : X) {
            predictions.push_back(predict(x));
        }
        return predictions;
    }

    std::vector<double> get_weights() const { return weights; }
    double get_bias() const { return bias; }
};

int main() {
    // Generate sample data
    std::random_device rd;
    std::mt19937 gen(rd());
    std::normal_distribution<double> noise(0, 0.1);

    std::vector<std::vector<double>> X;
    std::vector<double> y;

    for (int i = 0; i < 100; ++i) {
        double x = i * 0.02;
        std::vector<double> x_i = {x};
        double y_i = 2 * x + 1 + noise(gen);
        X.push_back(x_i);
        y.push_back(y_i);
    }

    // Create and train model
    LinearRegression model(0.01, 1000);
    model.fit(X, y);

    // Print results
    std::cout << "Weight: " << model.get_weights()[0] << std::endl;
    std::cout << "Bias: " << model.get_bias() << std::endl;

    return 0;
}


using System;
using System.Linq;
using System.Collections.Generic;

public class LinearRegression
{
    private double[] weights;
    private double bias;
    private readonly double learningRate;
    private readonly int nIterations;

    public LinearRegression(double learningRate = 0.01, int nIterations = 1000)
    {
        this.learningRate = learningRate;
        this.nIterations = nIterations;
    }

    public void Fit(double[][] X, double[] y)
    {
        int nSamples = X.Length;
        int nFeatures = X[0].Length;

        // Initialize parameters
        weights = new double[nFeatures];
        bias = 0.0;

        // Gradient descent
        for (int iter = 0; iter < nIterations; iter++)
        {
            // Forward pass
            var predictions = X.Select(x => Predict(x)).ToArray();

            // Calculate gradients
            var dw = new double[nFeatures];
            double db = 0.0;

            for (int i = 0; i < nSamples; i++)
            {
                double error = predictions[i] - y[i];
                for (int j = 0; j < nFeatures; j++)
                {
                    dw[j] += error * X[i][j];
                }
                db += error;
            }

            // Update parameters
            for (int j = 0; j < nFeatures; j++)
            {
                weights[j] -= learningRate * dw[j] / nSamples;
            }
            bias -= learningRate * db / nSamples;
        }
    }

    public double Predict(double[] X)
    {
        return X.Zip(weights, (x, w) => x * w).Sum() + bias;
    }

    public double[] GetWeights() => weights;
    public double GetBias() => bias;

    public static void Main()
    {
        // Generate sample data
        var random = new Random(0);
        var X = Enumerable.Range(0, 100)
            .Select(i => new[] { i * 0.02 })
            .ToArray();
        var y = X.Select(x => 2 * x[0] + 1 + random.NextDouble() * 0.1)
            .ToArray();

        // Create and train model
        var model = new LinearRegression(0.01, 1000);
        model.Fit(X, y);

        // Print results
        Console.WriteLine($"Weight: {model.GetWeights()[0]}");
        Console.WriteLine($"Bias: {model.GetBias()}");

        // Make predictions
        var X_test = new[] { new[] { 0.0 }, new[] { 1.0 } };
        foreach (var x in X_test)
        {
            Console.WriteLine($"Prediction for x={x[0]}: {model.Predict(x)}");
        }
    }
}

Quick Info

Linear Regression