Implement Swish Beta

Author: andrewdalpino

CHANGELOG.md

@@ -2,7 +2,7 @@
     - Switch back to original fork of Tensor
     - Added `maxBins` hyper-parameter to CART-based learners
     - Added stream Deduplicator extractor
-    - Added SiLU activation function from Extras package
+    - Added the Swish activation function
 
 - 1.2.2
     - Allow empty dataset objects in `stack()`

docs/neural-network/activation-functions/silu.md

@@ -0,0 +1,22 @@
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/Swish.php">[source]</a></span>
+
+# Swish
+Swish is a smooth and non-monotonic rectified activation function. The inputs are weighted by the [Sigmoid](sigmoid.md) activation function acting as a self-gating mechanism. In addition, the `beta` parameter allows you to adjust the gate such that you can interpolate between the ReLU function and the linear function as `beta` goes from 0 to infinity.
+
+## Parameters
+## Parameters
+| # | Name | Default | Type | Description |
+|---|---|---|---|---|
+| 1 | beta | 1.0 | float | The parameter that adjusts the slope of the sigmoid gating mechanism. |
+
+## Example
+```php
+use Rubix\ML\NeuralNet\ActivationFunctions\Swish;
+
+$activationFunction = new Swish(1.0);
+```
+
+### References
+>- S. Elwing et al. (2017). Sigmoid-Weighted Linear Units for Neural Network Function Approximation in Reinforcement Learning.
+>- P. Ramachandran er al. (2017). Swish: A Self-gated Activation Function.
+>- P. Ramachandran et al. (2017). Searching for Activation Functions.

docs/neural-network/activation-functions/swish.md

@@ -166,10 +166,10 @@ nav:
         - ReLU: neural-network/activation-functions/relu.md
         - SELU: neural-network/activation-functions/selu.md
         - Sigmoid: neural-network/activation-functions/sigmoid.md
-        - SiLU: neural-network/activation-functions/silu.md
         - Softmax: neural-network/activation-functions/softmax.md
         - Soft Plus: neural-network/activation-functions/soft-plus.md
         - Soft Sign: neural-network/activation-functions/softsign.md
+        - Swish: neural-network/activation-functions/swish.md
         - Thresholded ReLU: neural-network/activation-functions/thresholded-relu.md
       - Cost Functions:
         - Cross Entropy: neural-network/cost-functions/cross-entropy.md

mkdocs.yml

@@ -3,34 +3,53 @@
 namespace Rubix\ML\NeuralNet\ActivationFunctions;
 
 use Tensor\Matrix;
+use InvalidArgumentException;
 
 /**
- * SiLU
+ * Swish
  *
- * *Sigmoid-weighted Linear Unit* (SiLU) is a smooth and non-monotonic rectified activation function. The inputs
- * are weighted by the [Sigmoid](sigmoid.md) activation function acting as a self-gating mechanism. In addition,
- * an inherent global minimum functions as an implicit regularizer.
+ * Swish is a smooth and non-monotonic rectified activation function. The inputs are weighted by the [Sigmoid](sigmoid.md)
+ * activation function acting as a self-gating mechanism. In addition, the `beta` parameter allows you to adjust the gate
+ * such that you can interpolate between the ReLU function and the linear function as `beta` goes from 0 to infinity.
  *
  * References:
  * [1] S. Elwing et al. (2017). Sigmoid-Weighted Linear Units for Neural Network Function
  * Approximation in Reinforcement Learning.
- * [2] P. Ramachandran er al. (2017). Swish: A Self-gated Activation Function.
+ * [2] P. Ramachandran et al. (2017). Swish: A Self-gated Activation Function.
+ * [3] P. Ramachandran et al. (2017). Searching for Activation Functions.
  *
  * @category    Machine Learning
  * @package     Rubix/ML
  * @author      Andrew DalPino
  */
-class SiLU implements ActivationFunction
+class Swish implements ActivationFunction
 {
+    /**
+     * The parameter that adjusts the slope of the sigmoid gating mechanism.
+     *
+     * @var float
+     */
+    protected float $beta;
+
     /**
      * The sigmoid activation function.
      *
      * @var \Rubix\ML\NeuralNet\ActivationFunctions\Sigmoid
      */
-    protected $sigmoid;
+    protected \Rubix\ML\NeuralNet\ActivationFunctions\Sigmoid $sigmoid;
 
-    public function __construct()
+    /**
+     * @param float $beta
+     * @throws \InvalidArgumentException
+     */
+    public function __construct(float $beta = 1.0)
     {
+        if ($beta < 0.0) {
+            throw new InvalidArgumentException('Beta must be greater than'
+                . " 0, $beta given.");
+        }
+
+        $this->beta = $beta;
         $this->sigmoid = new Sigmoid();
     }
 
@@ -42,7 +61,8 @@ public function __construct()
      */
     public function compute(Matrix $z) : Matrix
     {
-        return $this->sigmoid->compute($z)->multiply($z);
+        return $this->sigmoid->compute($z->multiply($this->beta))
+            ->multiply($z);
     }
 
     /**
@@ -68,6 +88,6 @@ public function differentiate(Matrix $z, Matrix $computed) : Matrix
      */
     public function __toString() : string
     {
-        return 'SiLU';
+        return "Swish (beta: {$this->beta})";
     }
 }

src/NeuralNet/ActivationFunctions/SiLU.php renamed to src/NeuralNet/ActivationFunctions/Swish.php

@@ -3,19 +3,19 @@
 namespace Rubix\ML\Tests\NeuralNet\ActivationFunctions;
 
 use Tensor\Matrix;
-use Rubix\ML\NeuralNet\ActivationFunctions\SiLU;
+use Rubix\ML\NeuralNet\ActivationFunctions\Swish;
 use Rubix\ML\NeuralNet\ActivationFunctions\ActivationFunction;
 use PHPUnit\Framework\TestCase;
 use Generator;
 
 /**
  * @group ActivationFunctions
- * @covers \Rubix\ML\NeuralNet\ActivationFunctions\SiLU
+ * @covers \Rubix\ML\NeuralNet\ActivationFunctions\Swish
  */
-class SiLUTest extends TestCase
+class SwishTest extends TestCase
 {
     /**
-     * @var \Rubix\ML\NeuralNet\ActivationFunctions\SiLU
+     * @var \Rubix\ML\NeuralNet\ActivationFunctions\Swish
      */
     protected $activationFn;
 
@@ -24,15 +24,15 @@ class SiLUTest extends TestCase
      */
     protected function setUp() : void
     {
-        $this->activationFn = new SiLU();
+        $this->activationFn = new Swish(1.0);
     }
 
     /**
      * @test
      */
     public function build() : void
     {
-        $this->assertInstanceOf(SiLU::class, $this->activationFn);
+        $this->assertInstanceOf(Swish::class, $this->activationFn);
         $this->assertInstanceOf(ActivationFunction::class, $this->activationFn);
     }
 

tests/NeuralNet/ActivationFunctions/SiLUTest.php renamed to tests/NeuralNet/ActivationFunctions/SwishTest.php

@@ -21,7 +21,7 @@
 use Rubix\ML\Transformers\ZScaleStandardizer;
 use Rubix\ML\CrossValidation\Metrics\RSquared;
 use Rubix\ML\NeuralNet\CostFunctions\LeastSquares;
-use Rubix\ML\NeuralNet\ActivationFunctions\LeakyReLU;
+use Rubix\ML\NeuralNet\ActivationFunctions\Swish;
 use Rubix\ML\Exceptions\InvalidArgumentException;
 use Rubix\ML\Exceptions\RuntimeException;
 use PHPUnit\Framework\TestCase;
@@ -84,9 +84,9 @@ protected function setUp() : void
 
         $this->estimator = new MLPRegressor([
             new Dense(10),
-            new Activation(new LeakyReLU()),
+            new Activation(new Swish()),
             new Dense(10),
-            new Activation(new LeakyReLU()),
+            new Activation(new Swish()),
         ], 10, new Adam(0.01), 1e-4, 100, 1e-3, 3, 0.1, new LeastSquares(), new RMSE());
 
         $this->metric = new RSquared();
@@ -147,9 +147,9 @@ public function params() : void
         $expected = [
             'hidden layers' => [
                 new Dense(10),
-                new Activation(new LeakyReLU()),
+                new Activation(new Swish()),
                 new Dense(10),
-                new Activation(new LeakyReLU()),
+                new Activation(new Swish()),
             ],
             'batch size' => 10,
             'optimizer' => new Adam(0.01),