📘 Activation Functions in Neural Networks

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

# Define input range
x = np.linspace(-5, 5, 1000)

# TensorFlow activation functions
activations = {
    'sigmoid': tf.nn.sigmoid,
    'tanh': tf.nn.tanh,
    'relu': tf.nn.relu,
    'leaky_relu': lambda x: tf.nn.leaky_relu(x, alpha=0.01),
    'elu': tf.nn.elu,
    'swish': tf.nn.swish,
    'gelu': tf.nn.gelu,
    'softplus': tf.nn.softplus
}

# Plot activation functions
fig, axes = plt.subplots(2, 4, figsize=(16, 8))
axes = axes.ravel()

for i, (name, func) in enumerate(activations.items()):
    y = func(x).numpy()
    axes[i].plot(x, y, linewidth=2)
    axes[i].set_title(f'{name.upper()}')
    axes[i].grid(True, alpha=0.3)
    axes[i].axhline(y=0, color='k', linewidth=0.5)
    axes[i].axvline(x=0, color='k', linewidth=0.5)

plt.tight_layout()
plt.show()

# Example neural network with different activations
def create_model(activation):
    model = tf.keras.Sequential([
        tf.keras.layers.Dense(64, activation=activation, input_shape=(10,)),
        tf.keras.layers.Dense(32, activation=activation),
        tf.keras.layers.Dense(1, activation='sigmoid')  # Output layer
    ])
    return model

# Compare training with different activations
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

# Generate sample data
X, y = make_classification(n_samples=1000, n_features=10, n_classes=2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Test different activations
activation_results = {}
activations_to_test = ['relu', 'tanh', 'sigmoid', 'elu']

for activation in activations_to_test:
    print(f"Training with {activation} activation...")

    model = create_model(activation)
    model.compile(optimizer='adam', 
                  loss='binary_crossentropy', 
                  metrics=['accuracy'])

    history = model.fit(X_train, y_train, 
                       epochs=50, 
                       batch_size=32, 
                       validation_data=(X_test, y_test),
                       verbose=0)

    # Store results
    activation_results[activation] = {
        'history': history,
        'final_accuracy': history.history['val_accuracy'][-1]
    }

# Plot training curves
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))

for activation, results in activation_results.items():
    history = results['history']
    ax1.plot(history.history['loss'], label=f'{activation} - train')
    ax1.plot(history.history['val_loss'], label=f'{activation} - val', linestyle='--')

    ax2.plot(history.history['accuracy'], label=f'{activation} - train')
    ax2.plot(history.history['val_accuracy'], label=f'{activation} - val', linestyle='--')

ax1.set_title('Loss Curves')
ax1.set_xlabel('Epoch')
ax1.set_ylabel('Loss')
ax1.legend()
ax1.grid(True)

ax2.set_title('Accuracy Curves')
ax2.set_xlabel('Epoch')
ax2.set_ylabel('Accuracy')
ax2.legend()
ax2.grid(True)

plt.tight_layout()
plt.show()

# Print final accuracies
print("\nFinal Validation Accuracies:")
for activation, results in activation_results.items():
    print(f"{activation}: {results['final_accuracy']:.4f}")

import torch
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt

# Define activation functions in PyTorch
class ActivationShowcase(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x, activation_type):
        if activation_type == 'sigmoid':
            return torch.sigmoid(x)
        elif activation_type == 'tanh':
            return torch.tanh(x)
        elif activation_type == 'relu':
            return F.relu(x)
        elif activation_type == 'leaky_relu':
            return F.leaky_relu(x, negative_slope=0.01)
        elif activation_type == 'elu':
            return F.elu(x)
        elif activation_type == 'gelu':
            return F.gelu(x)
        elif activation_type == 'swish':
            return x * torch.sigmoid(x)
        else:
            return x

# Visualize derivatives
def compute_gradients():
    x = torch.linspace(-5, 5, 1000, requires_grad=True)
    showcase = ActivationShowcase()

    activations = ['sigmoid', 'tanh', 'relu', 'leaky_relu', 'elu', 'gelu']

    fig, axes = plt.subplots(2, 3, figsize=(15, 10))
    axes = axes.ravel()

    for i, activation in enumerate(activations):
        # Forward pass
        y = showcase(x, activation)

        # Compute gradients
        y.sum().backward(retain_graph=True)
        gradients = x.grad.clone()
        x.grad.zero_()

        # Plot function and its derivative
        axes[i].plot(x.detach().numpy(), y.detach().numpy(), 
                     label=f'{activation}', linewidth=2)
        axes[i].plot(x.detach().numpy(), gradients.numpy(), 
                     label=f'{activation} derivative', linewidth=2, linestyle='--')
        axes[i].set_title(f'{activation.upper()} and its derivative')
        axes[i].legend()
        axes[i].grid(True, alpha=0.3)
        axes[i].axhline(y=0, color='k', linewidth=0.5)
        axes[i].axvline(x=0, color='k', linewidth=0.5)

    plt.tight_layout()
    plt.show()

compute_gradients()

# Neural network with custom activation
class CustomNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, activation='relu'):
        super().__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.fc2 = nn.Linear(hidden_size, hidden_size)
        self.fc3 = nn.Linear(hidden_size, output_size)
        self.activation = activation
        self.showcase = ActivationShowcase()

    def forward(self, x):
        x = self.showcase(self.fc1(x), self.activation)
        x = self.showcase(self.fc2(x), self.activation)
        x = torch.sigmoid(self.fc3(x))  # Output activation
        return x

# Test gradient flow with different activations
def test_gradient_flow():
    # Create deep network
    input_size, hidden_size, output_size = 10, 128, 1
    activations = ['sigmoid', 'tanh', 'relu', 'leaky_relu']

    results = {}

    for activation in activations:
        print(f"Testing gradient flow with {activation}...")

        # Create model
        model = CustomNN(input_size, hidden_size, output_size, activation)

        # Create dummy data
        x = torch.randn(32, input_size)
        y = torch.randint(0, 2, (32, 1)).float()

        # Forward pass
        output = model(x)
        loss = F.binary_cross_entropy(output, y)

        # Backward pass
        loss.backward()

        # Collect gradient statistics
        gradients = []
        for param in model.parameters():
            if param.grad is not None:
                gradients.extend(param.grad.flatten().tolist())

        results[activation] = {
            'mean_grad': np.mean(np.abs(gradients)),
            'std_grad': np.std(gradients),
            'max_grad': np.max(np.abs(gradients))
        }

        # Clear gradients
        model.zero_grad()

    # Print results
    print("\nGradient Flow Analysis:")
    print("Activation | Mean |Grad| | Std Grad | Max |Grad|")
    print("-" * 50)
    for activation, stats in results.items():
        print(f"{activation:10} | {stats['mean_grad']:.6f} | {stats['std_grad']:.6f} | {stats['max_grad']:.6f}")

test_gradient_flow()

import numpy as np
import matplotlib.pyplot as plt

class ActivationFunctions:
    """Complete implementation of activation functions from scratch"""

    @staticmethod
    def sigmoid(x):
        """Sigmoid activation function"""
        # Clip x to prevent overflow
        x = np.clip(x, -500, 500)
        return 1 / (1 + np.exp(-x))

    @staticmethod
    def sigmoid_derivative(x):
        """Derivative of sigmoid function"""
        s = ActivationFunctions.sigmoid(x)
        return s * (1 - s)

    @staticmethod
    def tanh(x):
        """Hyperbolic tangent activation function"""
        return np.tanh(x)

    @staticmethod
    def tanh_derivative(x):
        """Derivative of tanh function"""
        return 1 - np.tanh(x) ** 2

    @staticmethod
    def relu(x):
        """ReLU activation function"""
        return np.maximum(0, x)

    @staticmethod
    def relu_derivative(x):
        """Derivative of ReLU function"""
        return (x > 0).astype(float)

    @staticmethod
    def leaky_relu(x, alpha=0.01):
        """Leaky ReLU activation function"""
        return np.where(x > 0, x, alpha * x)

    @staticmethod
    def leaky_relu_derivative(x, alpha=0.01):
        """Derivative of Leaky ReLU function"""
        return np.where(x > 0, 1, alpha)

    @staticmethod
    def elu(x, alpha=1.0):
        """ELU activation function"""
        return np.where(x > 0, x, alpha * (np.exp(x) - 1))

    @staticmethod
    def elu_derivative(x, alpha=1.0):
        """Derivative of ELU function"""
        return np.where(x > 0, 1, alpha * np.exp(x))

    @staticmethod
    def swish(x):
        """Swish activation function"""
        return x * ActivationFunctions.sigmoid(x)

    @staticmethod
    def swish_derivative(x):
        """Derivative of Swish function"""
        sigmoid_x = ActivationFunctions.sigmoid(x)
        return sigmoid_x + x * sigmoid_x * (1 - sigmoid_x)

    @staticmethod
    def softplus(x):
        """Softplus activation function"""
        # Use log(1 + exp(x)) but handle large values to prevent overflow
        return np.where(x > 20, x, np.log(1 + np.exp(x)))

    @staticmethod
    def softplus_derivative(x):
        """Derivative of Softplus function"""
        return ActivationFunctions.sigmoid(x)

    @staticmethod
    def gelu(x):
        """GELU activation function (approximation)"""
        return 0.5 * x * (1 + np.tanh(np.sqrt(2/np.pi) * (x + 0.044715 * x**3)))

    @staticmethod
    def gelu_derivative(x):
        """Derivative of GELU function (approximation)"""
        tanh_term = np.tanh(np.sqrt(2/np.pi) * (x + 0.044715 * x**3))
        sech_term = 1 - tanh_term**2
        return 0.5 * (1 + tanh_term) + 0.5 * x * sech_term * np.sqrt(2/np.pi) * (1 + 3 * 0.044715 * x**2)

class NeuralNetwork:
    """Simple neural network implementation with custom activation functions"""

    def __init__(self, layers, activation='relu'):
        """
        Initialize neural network

        Parameters:
        layers: list of layer sizes [input, hidden1, hidden2, ..., output]
        activation: activation function name
        """
        self.layers = layers
        self.activation = activation
        self.act_func = ActivationFunctions()

        # Initialize weights and biases
        self.weights = []
        self.biases = []

        for i in range(len(layers) - 1):
            # Xavier initialization
            w = np.random.randn(layers[i], layers[i+1]) * np.sqrt(2.0 / layers[i])
            b = np.zeros((1, layers[i+1]))
            self.weights.append(w)
            self.biases.append(b)

    def get_activation_function(self, name):
        """Get activation function and its derivative"""
        functions = {
            'sigmoid': (self.act_func.sigmoid, self.act_func.sigmoid_derivative),
            'tanh': (self.act_func.tanh, self.act_func.tanh_derivative),
            'relu': (self.act_func.relu, self.act_func.relu_derivative),
            'leaky_relu': (self.act_func.leaky_relu, self.act_func.leaky_relu_derivative),
            'elu': (self.act_func.elu, self.act_func.elu_derivative),
            'swish': (self.act_func.swish, self.act_func.swish_derivative)
        }
        return functions.get(name, (self.act_func.relu, self.act_func.relu_derivative))

    def forward(self, X):
        """Forward propagation"""
        self.activations = [X]
        self.z_values = []

        activation_func, _ = self.get_activation_function(self.activation)

        for i in range(len(self.weights)):
            # Linear transformation
            z = np.dot(self.activations[-1], self.weights[i]) + self.biases[i]
            self.z_values.append(z)

            # Apply activation function (except for output layer)
            if i < len(self.weights) - 1:
                a = activation_func(z)
            else:
                # Output layer - use sigmoid for binary classification
                a = self.act_func.sigmoid(z)

            self.activations.append(a)

        return self.activations[-1]

    def backward(self, X, y, learning_rate=0.01):
        """Backward propagation"""
        m = X.shape[0]

        _, activation_derivative = self.get_activation_function(self.activation)

        # Start from output layer
        dz = self.activations[-1] - y  # For sigmoid + BCE loss

        # Backpropagate through all layers
        for i in reversed(range(len(self.weights))):
            # Compute gradients
            dW = (1/m) * np.dot(self.activations[i].T, dz)
            db = (1/m) * np.sum(dz, axis=0, keepdims=True)

            # Update weights and biases
            self.weights[i] -= learning_rate * dW
            self.biases[i] -= learning_rate * db

            # Compute dz for previous layer (if not input layer)
            if i > 0:
                da_prev = np.dot(dz, self.weights[i].T)
                dz = da_prev * activation_derivative(self.z_values[i-1])

    def train(self, X, y, epochs=1000, learning_rate=0.01):
        """Train the neural network"""
        losses = []

        for epoch in range(epochs):
            # Forward propagation
            output = self.forward(X)

            # Compute loss (Binary Cross Entropy)
            loss = -np.mean(y * np.log(output + 1e-15) + (1 - y) * np.log(1 - output + 1e-15))
            losses.append(loss)

            # Backward propagation
            self.backward(X, y, learning_rate)

            if epoch % 100 == 0:
                print(f"Epoch {epoch}, Loss: {loss:.6f}")

        return losses

    def predict(self, X):
        """Make predictions"""
        return self.forward(X)

# Example usage and comparison
def compare_activations():
    """Compare different activation functions on a classification task"""

    # Generate sample data
    np.random.seed(42)
    from sklearn.datasets import make_classification

    X, y = make_classification(n_samples=1000, n_features=2, n_redundant=0, 
                             n_informative=2, n_clusters_per_class=1, random_state=42)
    y = y.reshape(-1, 1)

    # Normalize features
    X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)

    # Test different activation functions
    activations = ['sigmoid', 'tanh', 'relu', 'leaky_relu', 'elu', 'swish']
    results = {}

    for activation in activations:
        print(f"\nTraining with {activation} activation...")

        # Create and train network
        nn = NeuralNetwork([2, 10, 10, 1], activation=activation)
        losses = nn.train(X, y, epochs=500, learning_rate=0.1)

        # Final predictions
        predictions = nn.predict(X)
        accuracy = np.mean((predictions > 0.5) == y)

        results[activation] = {
            'losses': losses,
            'accuracy': accuracy,
            'final_loss': losses[-1]
        }

        print(f"Final accuracy: {accuracy:.4f}")

    # Plot training curves
    plt.figure(figsize=(15, 10))

    # Loss curves
    plt.subplot(2, 2, 1)
    for activation, result in results.items():
        plt.plot(result['losses'], label=activation)
    plt.title('Training Loss Curves')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.grid(True)

    # Final accuracies
    plt.subplot(2, 2, 2)
    activations_list = list(results.keys())
    accuracies = [results[act]['accuracy'] for act in activations_list]
    plt.bar(activations_list, accuracies)
    plt.title('Final Accuracies')
    plt.ylabel('Accuracy')
    plt.xticks(rotation=45)
    plt.grid(True, alpha=0.3)

    # Decision boundaries for best performing activation
    best_activation = max(results.keys(), key=lambda x: results[x]['accuracy'])
    print(f"\nBest performing activation: {best_activation}")

    # Plot decision boundary
    plt.subplot(2, 1, 2)

    # Create mesh
    h = 0.02
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))

    # Train best model
    best_nn = NeuralNetwork([2, 10, 10, 1], activation=best_activation)
    best_nn.train(X, y, epochs=500, learning_rate=0.1)

    # Predict on mesh
    mesh_points = np.c_[xx.ravel(), yy.ravel()]
    Z = best_nn.predict(mesh_points)
    Z = Z.reshape(xx.shape)

    # Plot
    plt.contourf(xx, yy, Z, levels=50, alpha=0.8, cmap='RdYlBu')
    scatter = plt.scatter(X[:, 0], X[:, 1], c=y.ravel(), cmap='RdYlBu', edgecolors='black')
    plt.colorbar(scatter)
    plt.title(f'Decision Boundary ({best_activation} activation)')
    plt.xlabel('Feature 1')
    plt.ylabel('Feature 2')

    plt.tight_layout()
    plt.show()

    return results

# Run comparison
if __name__ == "__main__":
    results = compare_activations()

import numpy as np
import matplotlib.pyplot as plt

# Create comprehensive visualization
def plot_activations_and_gradients():
    x = np.linspace(-5, 5, 1000)

    # Define activation functions
    def sigmoid(x):
        return 1 / (1 + np.exp(-np.clip(x, -500, 500)))

    def tanh(x):
        return np.tanh(x)

    def relu(x):
        return np.maximum(0, x)

    def leaky_relu(x, alpha=0.01):
        return np.where(x > 0, x, alpha * x)

    def elu(x, alpha=1.0):
        return np.where(x > 0, x, alpha * (np.exp(np.clip(x, -500, 500)) - 1))

    def swish(x):
        return x * sigmoid(x)

    # Define derivatives
    def sigmoid_grad(x):
        s = sigmoid(x)
        return s * (1 - s)

    def tanh_grad(x):
        return 1 - np.tanh(x)**2

    def relu_grad(x):
        return (x > 0).astype(float)

    def leaky_relu_grad(x, alpha=0.01):
        return np.where(x > 0, 1, alpha)

    def elu_grad(x, alpha=1.0):
        return np.where(x > 0, 1, alpha * np.exp(np.clip(x, -500, 500)))

    def swish_grad(x):
        s = sigmoid(x)
        return s + x * s * (1 - s)

    activations = [
        ('Sigmoid', sigmoid, sigmoid_grad, 'blue'),
        ('Tanh', tanh, tanh_grad, 'red'),
        ('ReLU', relu, relu_grad, 'green'),
        ('Leaky ReLU', leaky_relu, leaky_relu_grad, 'orange'),
        ('ELU', elu, elu_grad, 'purple'),
        ('Swish', swish, swish_grad, 'brown')
    ]

    fig, axes = plt.subplots(2, 3, figsize=(18, 12))
    axes = axes.ravel()

    for i, (name, func, grad_func, color) in enumerate(activations):
        y = func(x)
        dy = grad_func(x)

        ax = axes[i]
        ax.plot(x, y, label=f'{name}', color=color, linewidth=2)
        ax.plot(x, dy, label=f'{name} derivative', color=color, linewidth=2, linestyle='--', alpha=0.7)

        ax.set_title(f'{name} Activation Function')
        ax.legend()
        ax.grid(True, alpha=0.3)
        ax.axhline(y=0, color='black', linewidth=0.5)
        ax.axvline(x=0, color='black', linewidth=0.5)
        ax.set_xlabel('Input (x)')
        ax.set_ylabel('Output')

    plt.tight_layout()
    plt.suptitle('Activation Functions and Their Derivatives', fontsize=16, y=1.02)
    plt.show()

plot_activations_and_gradients()

from sklearn.datasets import load_breast_cancer, load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import pandas as pd

def comprehensive_activation_comparison():
    """Compare activation functions on real datasets"""

    # Load datasets
    datasets = {
        'Breast Cancer (Binary)': load_breast_cancer(),
        'Iris (Multi-class)': load_iris()
    }

    activations = ['relu', 'tanh', 'sigmoid', 'leaky_relu', 'elu', 'swish']
    results = {}

    for dataset_name, dataset in datasets.items():
        print(f"\n{'='*50}")
        print(f"Dataset: {dataset_name}")
        print(f"{'='*50}")

        X, y = dataset.data, dataset.target
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=0.2, random_state=42, stratify=y
        )

        # Standardize features
        scaler = StandardScaler()
        X_train_scaled = scaler.fit_transform(X_train)
        X_test_scaled = scaler.transform(X_test)

        dataset_results = {}

        for activation in activations:
            print(f"\nTesting {activation}...")

            # Create model architecture based on dataset
            if 'Binary' in dataset_name:
                # Binary classification
                model = tf.keras.Sequential([
                    tf.keras.layers.Dense(64, activation=activation, input_shape=(X_train.shape[1],)),
                    tf.keras.layers.Dropout(0.3),
                    tf.keras.layers.Dense(32, activation=activation),
                    tf.keras.layers.Dropout(0.3),
                    tf.keras.layers.Dense(1, activation='sigmoid')
                ])
                model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
                y_train_model, y_test_model = y_train, y_test
            else:
                # Multi-class classification
                model = tf.keras.Sequential([
                    tf.keras.layers.Dense(64, activation=activation, input_shape=(X_train.shape[1],)),
                    tf.keras.layers.Dropout(0.3),
                    tf.keras.layers.Dense(32, activation=activation),
                    tf.keras.layers.Dropout(0.3),
                    tf.keras.layers.Dense(len(np.unique(y)), activation='softmax')
                ])
                model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
                y_train_model, y_test_model = y_train, y_test

            # Train model
            history = model.fit(
                X_train_scaled, y_train_model,
                validation_data=(X_test_scaled, y_test_model),
                epochs=100,
                batch_size=32,
                verbose=0
            )

            # Evaluate
            test_loss, test_accuracy = model.evaluate(X_test_scaled, y_test_model, verbose=0)

            # Store results
            dataset_results[activation] = {
                'test_accuracy': test_accuracy,
                'test_loss': test_loss,
                'train_history': history.history,
                'convergence_epoch': np.argmin(history.history['val_loss']) + 1
            }

            print(f"  Test Accuracy: {test_accuracy:.4f}")
            print(f"  Test Loss: {test_loss:.4f}")
            print(f"  Convergence Epoch: {dataset_results[activation]['convergence_epoch']}")

        results[dataset_name] = dataset_results

        # Plot results for this dataset
        plot_dataset_results(dataset_name, dataset_results)

    # Summary comparison
    print_summary_results(results)

    return results

def plot_dataset_results(dataset_name, results):
    """Plot training curves and final metrics for a dataset"""

    fig, axes = plt.subplots(2, 2, figsize=(15, 10))
    fig.suptitle(f'Results for {dataset_name}', fontsize=16)

    # Training curves
    for activation, result in results.items():
        history = result['train_history']

        # Training loss
        axes[0, 0].plot(history['loss'], label=f'{activation}')
        axes[0, 1].plot(history['val_loss'], label=f'{activation}')
        axes[1, 0].plot(history['accuracy'], label=f'{activation}')
        axes[1, 1].plot(history['val_accuracy'], label=f'{activation}')

    axes[0, 0].set_title('Training Loss')
    axes[0, 0].set_ylabel('Loss')
    axes[0, 0].legend()
    axes[0, 0].grid(True)

    axes[0, 1].set_title('Validation Loss')
    axes[0, 1].set_ylabel('Loss')
    axes[0, 1].legend()
    axes[0, 1].grid(True)

    axes[1, 0].set_title('Training Accuracy')
    axes[1, 0].set_ylabel('Accuracy')
    axes[1, 0].set_xlabel('Epoch')
    axes[1, 0].legend()
    axes[1, 0].grid(True)

    axes[1, 1].set_title('Validation Accuracy')
    axes[1, 1].set_ylabel('Accuracy')
    axes[1, 1].set_xlabel('Epoch')
    axes[1, 1].legend()
    axes[1, 1].grid(True)

    plt.tight_layout()
    plt.show()

def print_summary_results(results):
    """Print summary comparison across all datasets"""

    print(f"\n{'='*80}")
    print("SUMMARY COMPARISON")
    print(f"{'='*80}")

    for dataset_name, dataset_results in results.items():
        print(f"\n{dataset_name}:")
        print("-" * (len(dataset_name) + 1))

        # Sort by test accuracy
        sorted_results = sorted(dataset_results.items(), 
                              key=lambda x: x[1]['test_accuracy'], 
                              reverse=True)

        print(f"{'Activation':<15} {'Test Acc':<10} {'Test Loss':<10} {'Convergence':<12}")
        print("-" * 55)

        for activation, result in sorted_results:
            print(f"{activation:<15} {result['test_accuracy']:<10.4f} "
                  f"{result['test_loss']:<10.4f} {result['convergence_epoch']:<12}")

# Run comprehensive comparison
# results = comprehensive_activation_comparison()

def analyze_gradient_flow():
    """Analyze how gradients flow through deep networks with different activations"""

    def create_deep_network(activation, depth=10):
        """Create a deep network for gradient flow analysis"""
        layers = []

        # Input layer
        layers.append(tf.keras.layers.Dense(64, activation=activation, input_shape=(100,)))

        # Hidden layers
        for _ in range(depth - 2):
            layers.append(tf.keras.layers.Dense(64, activation=activation))

        # Output layer
        layers.append(tf.keras.layers.Dense(1, activation='sigmoid'))

        return tf.keras.Sequential(layers)

    # Test different depths and activations
    activations = ['sigmoid', 'tanh', 'relu', 'leaky_relu', 'elu', 'swish']
    depths = [3, 5, 10, 15, 20]

    results = {}

    # Generate dummy data
    X = np.random.randn(1000, 100)
    y = np.random.randint(0, 2, 1000)

    for activation in activations:
        results[activation] = {}

        for depth in depths:
            print(f"Testing {activation} with depth {depth}")

            # Create model
            model = create_deep_network(activation, depth)
            model.compile(optimizer='adam', loss='binary_crossentropy')

            # Single forward-backward pass to analyze gradients
            with tf.GradientTape() as tape:
                predictions = model(X[:32])  # Small batch for analysis
                loss = tf.keras.losses.binary_crossentropy(y[:32], predictions)
                loss = tf.reduce_mean(loss)

            # Compute gradients
            gradients = tape.gradient(loss, model.trainable_variables)

            # Analyze gradient statistics
            gradient_norms = []
            layer_names = []

            for i, grad in enumerate(gradients):
                if grad is not None:
                    norm = tf.norm(grad).numpy()
                    gradient_norms.append(norm)
                    layer_names.append(f"Layer_{i//2 + 1}")  # Account for weights and biases

            # Store results
            results[activation][depth] = {
                'gradient_norms': gradient_norms,
                'mean_gradient_norm': np.mean(gradient_norms),
                'std_gradient_norm': np.std(gradient_norms),
                'min_gradient_norm': np.min(gradient_norms),
                'max_gradient_norm': np.max(gradient_norms)
            }

    # Plot results
    fig, axes = plt.subplots(2, 2, figsize=(15, 10))

    # Mean gradient norm vs depth
    for activation in activations:
        mean_norms = [results[activation][depth]['mean_gradient_norm'] for depth in depths]
        axes[0, 0].plot(depths, mean_norms, marker='o', label=activation)

    axes[0, 0].set_title('Mean Gradient Norm vs Network Depth')
    axes[0, 0].set_xlabel('Network Depth')
    axes[0, 0].set_ylabel('Mean Gradient Norm')
    axes[0, 0].legend()
    axes[0, 0].grid(True)
    axes[0, 0].set_yscale('log')

    # Gradient norm variance vs depth
    for activation in activations:
        std_norms = [results[activation][depth]['std_gradient_norm'] for depth in depths]
        axes[0, 1].plot(depths, std_norms, marker='o', label=activation)

    axes[0, 1].set_title('Gradient Norm Std vs Network Depth')
    axes[0, 1].set_xlabel('Network Depth')
    axes[0, 1].set_ylabel('Gradient Norm Std')
    axes[0, 1].legend()
    axes[0, 1].grid(True)
    axes[0, 1].set_yscale('log')

    # Min gradient norm (vanishing gradient indicator)
    for activation in activations:
        min_norms = [results[activation][depth]['min_gradient_norm'] for depth in depths]
        axes[1, 0].plot(depths, min_norms, marker='o', label=activation)

    axes[1, 0].set_title('Min Gradient Norm vs Network Depth')
    axes[1, 0].set_xlabel('Network Depth')
    axes[1, 0].set_ylabel('Min Gradient Norm')
    axes[1, 0].legend()
    axes[1, 0].grid(True)
    axes[1, 0].set_yscale('log')

    # Max gradient norm (exploding gradient indicator)
    for activation in activations:
        max_norms = [results[activation][depth]['max_gradient_norm'] for depth in depths]
        axes[1, 1].plot(depths, max_norms, marker='o', label=activation)

    axes[1, 1].set_title('Max Gradient Norm vs Network Depth')
    axes[1, 1].set_xlabel('Network Depth')
    axes[1, 1].set_ylabel('Max Gradient Norm')
    axes[1, 1].legend()
    axes[1, 1].grid(True)
    axes[1, 1].set_yscale('log')

    plt.tight_layout()
    plt.show()

    return results

# Run gradient flow analysis
# gradient_results = analyze_gradient_flow()