Added Activation Functions project files

Author: Agrover112

Scripts/Miscellaneous/Activation_Functions_from_scratch_in_Keras/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Agrover112
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Scripts/Miscellaneous/Activation_Functions_from_scratch_in_Keras/README.md

@@ -0,0 +1,40 @@
+# ActivationFunctions using Custom Layers in Keras
+Activation functions are an important are of deep learning research .Many new activation functions are being developed ,these include *bio-inspired* activtions, *purely mathematical activation functions* including others . Despite, such advancements we usually find ourselves using RELU and LeakyRELU commonly without using/thinking about others.
+In the following notebooks I showcase how easy/difficult it is to port an activation function using **Custom Layers in Keras and Tensorflow!**
+
+
+Link to main notebook --> [Activations.ipynb](https://github.com/Agrover112/ActivationFunctions/blob/master/src/Activation-Functions(GELU%2CSELU%2CELU%2CLeakyReLU%2CPRELU).ipynb)
+
+### Implemented activations:
+ 
+- LeakyReLu
+- ParametricReLu
+- Elu
+- SElu
+- GELU
+
+
+
+### Structure
+ ```  
+src
+|
+|-- Activations.ipynb
+|-- utils
+      |-- Utils.ipynb
+      |-- utils.py
+      
+references
+|
+|--Ref1
+|--Refn
+
+```
+
+###  Usage
+ ``` 
+ git clone  https://github.com/Agrover112/ActivationFunctions.git
+```
+
+### References
+- [References:D](https://github.com/Agrover112/ActivationFunctions/tree/master/references)

Scripts/Miscellaneous/Activation_Functions_from_scratch_in_Keras/references/README.md

@@ -0,0 +1,7 @@
+## References
+- SELU : [Self Normailizing Neural Networks](https://arxiv.org/abs/1706.02515)
+- GELU : [Gaussian Error Linear Units (GELUs)- Elu ](https://arxiv.org/abs/1606.08415)
+- PRELU : [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](https://arxiv.org/abs/1502.01852)
+- Leaky ReLU : [Rectifier Nonlinearities Improve Neural Network Acoustic Models](http://robotics.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf)
+    
+- [Activation Functions Blog](https://mlfromscratch.com/activation-functions-explained/#elu)

Scripts/Miscellaneous/Activation_Functions_from_scratch_in_Keras/src/Activation-Functions(GELU,SELU,ELU,LeakyReLU,PRELU).ipynb

@@ -0,0 +1 @@
+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Utils","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyMYHshcegvtTBQwBygO/eoj"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"_ImvYera1GfS","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1595963420315,"user_tz":-330,"elapsed":2543,"user":{"displayName":"Agrover112","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiMJACGAX3kCfRjB2hgzdG8w9zL1lAAKbPPMz0qLA=s64","userId":"09574164879083471944"}}},"source":["import tensorflow as tf\n","import numpy as np\n","import matplotlib.pyplot as plt\n","def load_data():\n","    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n","    x_train = np.reshape(x_train, (x_train.shape[0], 784))/255.\n","    x_test = np.reshape(x_test, (x_test.shape[0], 784))/255.\n","    y_train = tf.keras.utils.to_categorical(y_train)\n","    y_test = tf.keras.utils.to_categorical(y_test)\n","    return (x_train, y_train), (x_test, y_test)\n","\n","def plot_random_examples(x, y, p=None):\n","    indices = np.random.choice(range(0, x.shape[0]), 10)\n","    y = np.argmax(y, axis=1)\n","    if p is None:\n","        p = y\n","    plt.figure(figsize=(10, 5))\n","    for i, index in enumerate(indices):\n","        plt.subplot(2, 5, i+1)\n","        plt.imshow(x[index].reshape((28, 28)), cmap='binary')\n","        plt.xticks([])\n","        plt.yticks([])\n","        if y[index] == p[index]:\n","            col = 'g'\n","        else:\n","            col = 'r'\n","        plt.xlabel(str(p[index]), color=col)\n","    return plt\n","\n","def plot_results(history):\n","    history = history.history\n","    plt.figure(figsize=(12, 4))\n","    epochs = len(history['val_loss'])\n","    plt.subplot(1, 2, 1)\n","    plt.plot(range(epochs), history['val_loss'], label='Val Loss')\n","    plt.plot(range(epochs), history['loss'], label='Train Loss')\n","    plt.xticks(list(range(epochs)))\n","    plt.xlabel('Epochs')\n","    plt.ylabel('Loss')\n","    plt.legend()\n","    plt.subplot(1, 2, 2)\n","    plt.plot(range(epochs), history['val_accuracy'], label='Val Acc')\n","    plt.plot(range(epochs), history['accuracy'], label='Acc')\n","    plt.xticks(list(range(epochs)))\n","    plt.xlabel('Epochs')\n","    plt.ylabel('Accuracy')\n","    plt.legend()\n","    return plt"],"execution_count":1,"outputs":[]},{"cell_type":"code","metadata":{"id":"iRwkOk0p1SPt","colab_type":"code","colab":{}},"source":[""],"execution_count":null,"outputs":[]}]}

Scripts/Miscellaneous/Activation_Functions_from_scratch_in_Keras/src/utils/Utils

@@ -0,0 +1,49 @@
+import tensorflow as tf
+import numpy as np
+import matplotlib.pyplot as plt
+
+def load_data():
+    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
+    x_train = np.reshape(x_train, (x_train.shape[0], 784))/255.
+    x_test = np.reshape(x_test, (x_test.shape[0], 784))/255.
+    y_train = tf.keras.utils.to_categorical(y_train)
+    y_test = tf.keras.utils.to_categorical(y_test)
+    return (x_train, y_train), (x_test, y_test)
+
+def plot_random_examples(x, y, p=None):
+    indices = np.random.choice(range(0, x.shape[0]), 10)
+    y = np.argmax(y, axis=1)
+    if p is None:
+        p = y
+    plt.figure(figsize=(10, 5))
+    for i, index in enumerate(indices):
+        plt.subplot(2, 5, i+1)
+        plt.imshow(x[index].reshape((28, 28)), cmap='binary')
+        plt.xticks([])
+        plt.yticks([])
+        if y[index] == p[index]:
+            col = 'g'
+        else:
+            col = 'r'
+        plt.xlabel(str(p[index]), color=col)
+    return plt
+
+def plot_results(history):
+    history = history.history
+    plt.figure(figsize=(12, 4))
+    epochs = len(history['val_loss'])
+    plt.subplot(1, 2, 1)
+    plt.plot(range(epochs), history['val_loss'], label='Val Loss')
+    plt.plot(range(epochs), history['loss'], label='Train Loss')
+    plt.xticks(list(range(epochs)))
+    plt.xlabel('Epochs')
+    plt.ylabel('Loss')
+    plt.legend()
+    plt.subplot(1, 2, 2)
+    plt.plot(range(epochs), history['val_accuracy'], label='Val Acc')
+    plt.plot(range(epochs), history['accuracy'], label='Acc')
+    plt.xticks(list(range(epochs)))
+    plt.xlabel('Epochs')
+    plt.ylabel('Accuracy')
+    plt.legend()
+    return plt