JavaScript Numerical Computational Series
Binary Classification using Logistic Regression w/ 4 optimizers and 2 loss functions.
In this post, basic, momentum, rmsprop, and adam optimizers are demonstrated for binary classification.
<!DOCTYPE>
<html>
<head></head>
<body>
<div class="javascript_css"></div>
<script>
function functionName0() {
var labelElement = document.createElement("label");
labelElement.setAttribute("id", "label0");
labelElement.style.position = "absolute";
labelElement.style.top = "60px";
labelElement.style.left = "0px";
labelElement.style.display = "flex";
labelElement.style.flexDirection = "column";
labelElement.style.gap = "2px";
labelElement.innerText = "Select an optimizer";
document.getElementsByClassName("javascript_css")[0].appendChild(labelElement);
var selectElement = document.createElement("select");
selectElement.setAttribute("name", "select0_name");
selectElement.setAttribute("id", "select0");
selectElement.style.background = "rgb(253, 255, 244)";
labelElement.appendChild(selectElement);
var option0 = ["basic", "momentum", "rmsprop", "adam"];
for (var i=0; i<option0.length; i++) {
var optionElement = document.createElement("option");
optionElement.setAttribute("value", option0.at(i));
optionElement.setAttribute("id", "option0_"+i);
optionElement.style.position = "absolute";
optionElement.style.width = "50px";
optionElement.style.height = "50px";
optionElement.style.borderRadius = "50%";
optionElement.style.border = "5px solid rgb(255, 255, 250)";
optionElement.style.background = "rgb(253, 255, 244)";
optionElement.style.boxShadow = "0 0 0 75px rgba(255, 255, 250, 0.3)";
optionElement.style.padding = 0;
optionElement.style.display = "grid"; // place checkmark in center
optionElement.style.placeItems = "center"; // place checkmark in center
optionElement.style.overflow = "visible";
optionElement.style.top = "50%";
optionElement.style.left = "50%";
optionElement.innerText = option0.at(i);
selectElement.appendChild(optionElement);
}
var labelElement = document.createElement("label");
labelElement.setAttribute("id", "label1");
labelElement.style.position = "absolute";
labelElement.style.top = "60px";
labelElement.style.left = "150px";
labelElement.style.display = "flex";
labelElement.style.flexDirection = "column";
labelElement.style.gap = "2px";
labelElement.innerText = "Select a loss function";
document.getElementsByClassName("javascript_css")[0].appendChild(labelElement);
var selectElement = document.createElement("select");
selectElement.setAttribute("name", "select1_name");
selectElement.setAttribute("id", "select1");
selectElement.style.background = "rgb(253, 255, 244)";
labelElement.appendChild(selectElement);
var option1 = ["none", "L2"];
for (var i=0; i<option1.length; i++) {
var optionElement = document.createElement("option");
optionElement.setAttribute("value", option1.at(i));
optionElement.setAttribute("id", "option1_"+i);
optionElement.style.position = "absolute";
optionElement.style.width = "50px";
optionElement.style.height = "50px";
optionElement.style.borderRadius = "50%";
optionElement.style.border = "5px solid rgb(255, 255, 250)";
optionElement.style.background = "rgb(253, 255, 244)";
optionElement.style.boxShadow = "0 0 0 75px rgba(255, 255, 250, 0.3)";
optionElement.style.padding = 0;
optionElement.style.display = "grid"; // place checkmark in center
optionElement.style.placeItems = "center";
optionElement.style.overflow = "visible";
optionElement.style.top = "50%";
optionElement.style.left = "50%";
optionElement.innerText = option1.at(i);
selectElement.appendChild(optionElement);
}
var svgElement = document.createElement("svg");
svgElement.setAttribute("id", "svg0");
svgElement.setAttribute("class", "svg_simple");
svgElement.setAttribute("onclick", "processEvent_click()");
svgElement.style.position = "absolute";
svgElement.style.top = "140px";
svgElement.style.left = "0px";
svgElement.style.width = "200px";
svgElement.style.height = "200px";
svgElement.style.border = "1px solid gray";
svgElement.style.borderRadius = "2px";
document.getElementsByClassName("javascript_css")[0].appendChild(svgElement);
var pElement = document.createElement("p");
pElement.setAttribute("id", "p0");
pElement.setAttribute("class", "p_simple");
pElement.style.position = "absolute";
pElement.style.top = "340px";
pElement.style.width = "300px";
pElement.style.height = "100px";
pElement.style.border = "1px solid gray";
pElement.style.borderRadius = "2px";
pElement.style.backgroundColor= "hsl(60 100% 90%)";
pElement.style.overflow = "scroll";
document.getElementsByClassName("javascript_css")[0].appendChild(pElement);
}
function processEvent_click() {
// Create categorical data.
var r = 30;
var c = 2;
var X = Array.from({length:r}, (v0,i0) => {
if (i0 < parseInt(r/2)) {
return Array.from({length:c}, (v1,i1) => { if (i1 == 0) { return Math.random() + 2; } else { return Math.random() + 1; }; });
} else {
return Array.from({length:c}, (v1,i1) => { if (i1 == 0) { return Math.random() + 10; } else { return Math.random() + 5; }; });
}
});
var y = Array.from({length:r}, (v,i) => {
if (i < parseInt(r/2)) {
return 0;
} else {
return 1;
}
});
document.getElementById("p0"). innerHTML+= "X: "+X+"<br>";
document.getElementById("p0"). innerHTML+= "y: "+y+"<br>";
var optimizer = document.getElementById("select0").value;
var lossFunction = document.getElementById("select1").value;
var [y_p, J, w, b] = gradient_descent(X, y, 0.001, 20, optimizer, lossFunction);
document.getElementById("p0"). innerHTML+= "y_p decimal: "+y_p+"<br>";
// console.log("J: ", J);
// console.log("w: ", w);
// console.log("b: ", b);
// Make sigmoid output a binary integer.
y_p = y_p.map((v,i) => { return (v > 0.5) ? 1 : 0; });
document.getElementById("p0"). innerHTML+= "y_p binary integer: "+y_p+"<br>";
var accuracy = y.map((v,i) => { return (v == y_p.at(i)) ? 1 : 0; }).reduce((a,c) => a+c, 0)/y.length;
document.getElementById("p0"). innerHTML+= "accuracy: "+accuracy+"<br>";
}
</script>
<script>
function plotter_2darray(x, y, this_id) {
// Plotting information.
var gapX = 1;
var gapY = 10;
var radius = 2;
var width = parseInt(document.getElementById(this_id).style.width.split("px")[0]);
var height = parseInt(document.getElementById(this_id).style.height.split("px")[0]);
var offsetY = parseInt(Math.floor(height/2));
var y_maximum = y.map((v,i) => { return Math.abs(v); }).sort().at(-1);
// document.getElementById("div0").innerHTML += "y_maximum: "+y_maximum+"<br>";
var mult = 1; // Math.floor(offsetY/y_maximum);
// document.getElementById("div0").innerHTML += "mult: "+mult+"<br>";
y = y.map((v,i) => { return -mult*v; });
// document.getElementById("div0").innerHTML += "y: "+y+"<br>";
var tot_html_circle_string = "";
for (var i=0; i<x.length; i++) {
tot_html_circle_string = tot_html_circle_string + `<circle id="circle${i}" class="circle_simple" cx="${x.at(i)*gapX}" cy="${y.at(i)*gapY+offsetY}" r="${radius}" fill="seagreen" stroke="seagreen"/>`;
if (i > 0) {
tot_html_circle_string = tot_html_circle_string + `<path id="line${i-1}" class="line_simple" d="M ${x.at(i-1)*gapX} ${y.at(i-1)*gapY+offsetY} L ${x.at(i)*gapX} ${y.at(i)*gapY+offsetY}" fill="transparent" stroke="seagreen" stroke-width="3"/>`;
}
}
var html_string = `<svg id="svg_0" class="svg_simple" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${width} ${height}">${tot_html_circle_string}</svg>`;
var url_string = `url(https://medium.com/'data:image/svg+xml;utf8,${html_string}')`;
document.getElementById(this_id).style.backgroundImage = url_string;
}
</script>
<script>
const dot = (arr0, arr1) => {
try {
arr0.at(0).length;
} catch (err) {
arr0 = [arr0];
}
var arr0_cols = arr0.at(0).length;
var arr1_rows = arr1.length;
if (arr0_cols == arr1_rows) {
/*Matrix multiplication*/
var arr0_rows = arr0.length;
var arr1_cols = arr1.at(0).length;
var dot = Array.from({length: arr0_rows}, (v0,i0) => { return Array.from({length: arr1_cols}, (v1,i1) => { return 0; }); });
arr0.map((row,r) => {
for (var c=0; c<arr1_cols; c++) {
/*Select columns.*/
var col = arr1.map((row,irow) => { return row.map((v,i) => { return (c == i) ? v : ""; }).filter((x) => x != ""); });
dot[r][c] = row.map((v,i) => { return v*col.at(i); }).reduce((a,cum) => a+cum, 0);
}
});
return dot;
} else {
return "Array dimensions are not correct for multiplication";
}
}
const add = (arr0, arr1) => {
function reshape(r, c, arrflat) {
var out = [];
for (var i=0; i<r; i++) {
var row = arrflat.slice(i*c, (i*c)+c);
out.push(row);
}
return out;
}
var r = arr0.length;
try { var c = arr0.at(0).length; } catch (err) { var c = 1; }
/*console.log("r,c: ", [r,c]);*/
if (arr0.flat(Infinity).length == arr1.flat(Infinity).length) {
/*Same length array element-wise multiplication.*/
arr1 = arr1.flat(Infinity);
var arrflat = arr0.flat(Infinity).map((v,i) => { return Number(Number(v)+Number(arr1.at(i))); });
var out = reshape(r, c, arrflat);
} else if (arr0.flat(Infinity).length == 1 || arr1.flat(Infinity).length == 1) {
/*Scalar and array multiplication.*/
if (arr0.flat(Infinity).length == 1) { var scalar = arr0.flat(Infinity); var arr = arr1.flat(Infinity); } else { var scalar = arr1.flat(Infinity); var arr = arr0.flat(Infinity); }
var arrflat = arr.map((v,i) => { return Number(Number(v)+Number(scalar)); });
var out = reshape(r, c, arrflat);
} else {
var out = "Arrays can not be added; both arrays must be the same size, or one array must be a scalar.";
}
return out;
}
function standardization(x) {
var avg = x.reduce((a,c) => a+c, 0)/x.length;
var standard_deviation = Math.pow(x.map((v,i) => { return Math.pow(v-avg, 2); }).reduce((a,c) => a+c, 0)/x.length, 0.5);
return x.map((v,i) => { return (v - avg)/standard_deviation; });
}
function sigmoid(arr) {
return arr.map((v,i) => { return [1/(1 + Math.exp(-v))]; });
}
function predict_logistic(X, w, b) {
var y_hat = add(dot(X, w), b);
// console.log("y_hat before scaling: ", y_hat);
// Normalize: make values maximally spread-out from [0, 1], so that sigmoid can be bounded from [0.5, 1].
// var minimum = Math.min(...y_hat.flat(Infinity));
// y_hat = y_hat.map((v,i) => { return v-minimum; });
// var maximum = Math.max(...y_hat.flat(Infinity));
// y_hat = y_hat.map((v,i) => { return v/maximum; });
// console.log("y_hat after scaling: ", y_hat);
// OR
// Standardize: make values maximally spread-out from [-1, 1], so that sigmoid can be bounded from [0, 1].
y_hat = standardization(y_hat.flat(Infinity));
// console.log("y_hat standardization: ", y_hat);
var arr = y_hat.flat(Infinity).map((v,i) => { return Math.abs(v); });
var maximum = Math.max(...arr);
y_hat = y_hat.map((v,i) => { return v/maximum; });
// console.log("y_hat after scaling: ", y_hat);
var y_hat_sigmoid = sigmoid(y_hat);
// console.log("y_hat_sigmoid: ", y_hat_sigmoid);
return y_hat_sigmoid;
}
function log_baseNum(scalar, base_number) {
return Math.log10(scalar)/Math.log10(base_number);
}
function partial_derivative_logistic(X, y_p, y) {
var r = X.length;
var c = X.at(0).length;
var y_p = y_p.flat(Infinity); // size [r,1]
var y = y.flat(Infinity); // size [r,1]
// dJ(w,b)/db = -1/N sum_{i=1}^{N} [ln(y_p_i) + (1-ln(y_p_i))]
var sum_dj_db = 0;
for (var i=0; i<r; i++) {
var ln_y_p_i = log_baseNum(y_p.at(i), 2.71828);
var sum_dj_db = sum_dj_db + ln_y_p_i + (1-ln_y_p_i);
}
// dJ(w,b)/dw = -1/N sum_{i=1}^{N} [(y_i/y_p_i) - ((1-y_i)/(1-y_p_i))]
var sum_dj_dw = 0;
for (var i=0; i<r; i++) {
var value = (y.at(i)/y_p.at(i)) - ( (1-y.at(i)) / (1-y_p.at(i)) );
var sum_dj_dw = sum_dj_dw + value;
}
var dj_dw = -(1/r) * sum_dj_dw;
var dj_db = -(1/r) * sum_dj_db;
return [dj_dw, dj_db];
}
function loss_logistic_regression_way1(y, y_p) {
var small_floating_point_number = 1/(10**Math.floor(6+Math.random()*20));
// console.log("small_floating_point_number: ", small_floating_point_number);
return -(1/y.length)* y.map((v,i) => {
// Prevent NaN.
if (y_p.at(i) == 0) {
var y_p0 = y_p.at(i) + small_floating_point_number; // Add a really small number to zero, so log(almost zero) will not give -Infinity.
} else {
var y_p0 = y_p.at(i);
}
if ((1-y_p.at(i)) == 0) {
var y_p1 = (1 - y_p.at(i)) + small_floating_point_number; // Add a really small number to zero.
} else {
var y_p1 = (1 - y_p.at(i));
}
return v*log_baseNum(y_p0, 2.71828) + (1-v)*log_baseNum(y_p1, 2.71828); }).reduce((a,c) => a+c, 0);
}
function loss_logistic_regression_w_L2_regularization(y, y_p, w, lambda) {
var small_floating_point_number = 1/(10**Math.floor(6+Math.random()*20));
// console.log("small_floating_point_number: ", small_floating_point_number);
var regularization = (lambda/(2*y.length)) * w.flat(Infinity).map((v,i) => { return v*v; }).reduce((a,c) => a+c, 0);
var loss = -(1/y.length)* y.map((v,i) => {
// Prevent NaN.
if (y_p.at(i) == 0) {
var y_p0 = y_p.at(i) + small_floating_point_number; // Add a really small number to zero, so log(almost zero) will not give -Infinity.
} else {
var y_p0 = y_p.at(i);
}
if ((1-y_p.at(i)) == 0) {
var y_p1 = (1 - y_p.at(i)) + small_floating_point_number; // Add a really small number to zero.
} else {
var y_p1 = (1 - y_p.at(i));
}
return v*log_baseNum(y_p0, 2.71828) + (1-v)*log_baseNum(y_p1, 2.71828); }).reduce((a,c) => a+c, 0);
return loss + regularization;
}
function gradient_descent(X, y, lr, iterations, optimizer, regularization) {
var r = X.length;
var c = X.at(0).length;
if (optimizer == "basic") {
var w = Array.from({length:c}, (v,i) => { return [Math.random()]; });
var b = Array.from({length:r}, (v,i) => { return [Math.random()]; });
} else if (optimizer == "momentum") {
var w = Array.from({length:c}, (v,i) => { return [Math.random()]; });
var b = Array.from({length:r}, (v,i) => { return [Math.random()]; });
var v_dj_dw = Array.from({length:c}, (v,i) => { return [Math.random()]; });
var v_dj_db = Array.from({length:r}, (v,i) => { return [Math.random()]; });
} else if (optimizer == "rmsprop") {
var w = Array.from({length:c}, (v,i) => { return [Math.random()]; });
var b = Array.from({length:r}, (v,i) => { return [Math.random()]; });
var s_dj_dw = Array.from({length:c}, (v,i) => { return [Math.random()]; });
var s_dj_db = Array.from({length:r}, (v,i) => { return [Math.random()]; });
} else if (optimizer == "adam") {
var w = Array.from({length:c}, (v,i) => { return [Math.random()]; });
var b = Array.from({length:r}, (v,i) => { return [Math.random()]; });
var v_dj_dw = Array.from({length:c}, (v,i) => { return [Math.random()]; });
var v_dj_db = Array.from({length:r}, (v,i) => { return [Math.random()]; });
var s_dj_dw = Array.from({length:c}, (v,i) => { return [Math.random()]; });
var s_dj_db = Array.from({length:r}, (v,i) => { return [Math.random()]; });
}
var y_p = predict_logistic(X, w, b);
document.getElementById("p0"). innerHTML+= "y_p: "+y_p+"<br>";
[dj_dw, dj_db] = partial_derivative_logistic(X, y_p, y);
// document.getElementById("p0"). innerHTML+= "dj_dw: "+dj_dw+"<br>";
// document.getElementById("p0"). innerHTML+= "dj_db: "+dj_db+"<br>";
if (regularization == "none") {
var J = [loss_logistic_regression_way1(y, y_p)];
} else if (regularization == "L2") {
var lambda = 0.001;
var J = [loss_logistic_regression_w_L2_regularization(y, y_p, w, lambda)];
}
// document.getElementById("p0"). innerHTML+= "J: "+J+"<br>";
for (var i=0; i<iterations; i++) {
if (optimizer == "basic") {
w = w.map((v,ii) => { return [v-(dj_dw*lr)]; });
b = b.map((v,ii) => { return [v-(dj_db*lr)]; });
// OR
// b = Array.from({length:r}, (v,ii) => { return [0]; });
} else if (optimizer == "momentum") {
var beta1 = 0.9;
v_dj_dw = v_dj_dw.map((v,ii) => { return beta1*v + (1-beta1)*dj_dw; });
document.getElementById("p0"). innerHTML+= "v_dj_dw: "+v_dj_dw+"<br>";
w = w.map((v,ii) => { return [v-(v_dj_dw.at(ii)*lr)]; });
document.getElementById("p0"). innerHTML+= "w: "+w+"<br>";
v_dj_db = v_dj_db.map((v,ii) => { return beta1*v + (1-beta1)*dj_db; });
b = b.map((v,ii) => { return [v-(v_dj_db.at(ii)*lr)]; });
} else if (optimizer == "rmsprop") {
var beta2 = 0.999;
var epsilon = 0.001;
s_dj_dw = s_dj_dw.map((v,ii) => { return beta2*v + (1-beta2)*(dj_dw*dj_dw); });
//document.getElementById("p0"). innerHTML+= "s_dj_dw: "+s_dj_dw+"<br>";
w = w.map((v,ii) => { return [v-(lr*(dj_dw/(Math.sqrt(s_dj_dw.at(ii))+epsilon)))]; });
s_dj_db = s_dj_db.map((v,ii) => { return beta2*v + (1-beta2)*(dj_db*dj_db); });
//document.getElementById("p0"). innerHTML+= "s_dj_db: "+s_dj_db+"<br>";
b = b.map((v,ii) => { return [v-(lr*(dj_db/(Math.sqrt(s_dj_db.at(ii))+epsilon)))]; });
} else if (optimizer == "adam") {
var beta1 = 0.9;
var beta2 = 0.999;
var epsilon = 0.001;
v_dj_dw = v_dj_dw.map((v,ii) => { return beta1*v + (1-beta1)*dj_dw; });
s_dj_dw = s_dj_dw.map((v,ii) => { return beta2*v + (1-beta2)*(dj_dw*dj_dw); });
//document.getElementById("p0"). innerHTML+= "s_dj_dw: "+s_dj_dw+"<br>";
var v_dj_dw_hat = v_dj_dw.map((v,ii) => { return (i == 0) ? v/(1-Math.pow(beta1,0.9)) : v/(1-Math.pow(beta1,i)); }); // With bias correction.
//document.getElementById("p0"). innerHTML+= "v_dj_dw_hat: "+v_dj_dw_hat+"<br>";
var s_dj_dw_hat = s_dj_dw.map((v,ii) => { return (i == 0) ? v/(1-Math.pow(beta2,0.9)) : v/(1-Math.pow(beta2,i)); }); // With bias correction.
w = w.map((v,ii) => { return [v-(lr*(v_dj_dw_hat.at(ii)/(Math.sqrt(s_dj_dw_hat.at(ii))+epsilon)))]; });
v_dj_db = v_dj_db.map((v,ii) => { return beta1*v + (1-beta1)*dj_db; });
s_dj_db = s_dj_db.map((v,ii) => { return beta2*v + (1-beta2)*(dj_db*dj_db); });
var v_dj_db_hat = v_dj_db.map((v,ii) => { return (i == 0) ? v/(1-Math.pow(beta1,0.9)) : v/(1-Math.pow(beta1,i)); }); // With bias correction.
var s_dj_db_hat = s_dj_db.map((v,ii) => { return (i == 0) ? v/(1-Math.pow(beta2,0.9)) : v/(1-Math.pow(beta2,i)); }); // With bias correction.
b = b.map((v,ii) => { return [v-(lr*(v_dj_db_hat.at(ii)/(Math.sqrt(s_dj_db_hat.at(ii))+epsilon)))]; });
}
y_p = predict_logistic(X, w, b);
[dj_dw, dj_db] = partial_derivative_logistic(X, y_p, y);
// document.getElementById("p0"). innerHTML+= "dj_dw: "+dj_dw+"<br>";
// document.getElementById("p0"). innerHTML+= "dj_db: "+dj_db+"<br>";
if (regularization == "none") {
J.push(loss_logistic_regression_way1(y, y_p));
} else if (regularization == "L2") {
J.push(loss_logistic_regression_w_L2_regularization(y, y_p, w, lambda));
}
} // end of for
document.getElementById("p0"). innerHTML+= "J: "+J+"<br>";
var xx = Array.from({length:J.length}, (v,i) => { return i; });
plotter_2darray(xx, J, "svg0");
return [y_p, J, w, b];
}
</script>
<script>
topbar();
function topbar() {
document.getElementsByClassName("javascript_css")[0].style.position = "absolute";
document.getElementsByClassName("javascript_css")[0].style.width = "300px";
document.getElementsByClassName("javascript_css")[0].style.height = "50px";
document.getElementsByClassName("javascript_css")[0].style.border = "1px solid gray";
document.getElementsByClassName("javascript_css")[0].style.borderRadius = "2px";
document.getElementsByClassName("javascript_css")[0].innerHTML = "Optimizer and Loss function.";
functionName0();
}
</script>
</body>
</html>Good Luck with your workflows and Happy Practicing! 👋
Get Practicing DatScy’s stories in your inbox
Join Medium for free to get updates from this writer.









