Add C++ Micrograd initial commit

README.md

@@ -1,2 +1,30 @@
-# ml
-
+# General ML Projects
+
+This repository is more of a general purpose place for me to store any general
+machine learning projects that I've done over time. 
+
+
+## Prerequisites
+If using Python, by default the following packages should be installed:
+
+You can either have the packages installed globally:
+
+```sh
+pip install numpy pandas matplotlib notebook torch torchvision torchaudio
+```
+
+Alternatively you can do this in virtual environment:
+```sh
+python -m venv venv
+source venv/bin/activate
+pip install numpy pandas matplotlib notebook torch torchvision torchaudio
+```
+
+This repository will assume it's installed globally, but we also include a
+`requirements.txt` to be precise of which versions we expect.
+
+# Projects
+
+The first few projects will follow along Andrej Karpathy's
+[Neural Networks: Zero to Hero](https://www.youtube.com/playlist?list=PLAqhIrjkxbuWI23v9cThsA9GvCAUhRvKZ)
+playlist and then will continue from there.

micrograd/README.md

@@ -0,0 +1,3 @@
+# micrograd
+
+An exercise implement micrograd in C++.

micrograd/main.cpp

@@ -0,0 +1,35 @@
+#include "value.h"
+
+#include <iostream>
+
+int main() {
+  Value<double> w1(-3.0, "w1");
+  Value<double> x1(2.0, "x1");
+
+  Value<double> w2(1.0, "w2");
+  Value<double> x2(0.0, "x2");
+
+  Value<double> b(6.8813735870195432, "b");
+
+
+  Value<double> w1x1 = w1 * x1;
+  w1x1.SetName("w1x1");
+  Value<double> w2x2 = w2 * x2;
+  w2x2.SetName("w2x2");
+
+
+  Value<double> w1x1w2x2 = w1x1 + w2x2;
+  w1x1w2x2.SetName("w1x1w2x2");
+
+
+  Value<double> n = w1x1w2x2 + b;
+  n.SetName("n");
+
+
+  Value<double> o = n.Tanh();
+  o.SetName("o");
+
+  o.Backward();
+
+  return 0;
+}

micrograd/value.h

@@ -0,0 +1,158 @@
+#ifndef VALUE_H
+
+#include <cmath>
+#include <functional>
+#include <iostream>
+#include <queue>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+template <typename T>
+class Value {
+ public:
+  Value(
+      T data, std::string op, std::vector<Value<T>*> children,
+      std::string name = "")
+      : data_(data), grad_(0), op_(op), children_(children), name_(name) {}
+
+  Value(T data, std::string name) : data_(data), grad_(0), op_(""), name_(name) {}
+  Value(T data) : data_(data), grad_(0), op_(""), name_("") {}
+
+  friend std::ostream& operator<<(std::ostream& os, const Value& val) {
+    os << val.name_ << " = " << val.data_ << ", grad = " << val.grad_
+       << ", op = " << val.op_ << " " << (val.backwards_ != nullptr) << std::endl;
+    return os;
+  }
+
+  void SetName(std::string name) { name_ = name; }
+
+  Value operator+(Value& other) {
+    std::vector<Value<T>*> children = {this, &other};
+    auto parent = Value(data_ + other.data_, "+", children);
+
+    parent.backwards_ = [&parent, &other, this]() {
+      this->grad_ += parent.grad_;
+      other.grad_ += parent.grad_;
+    };
+
+    return parent;
+  }
+
+  Value operator+(T scalar) {
+    return *this + Value(scalar);
+  }
+
+  Value operator*(Value& other) {
+    std::vector<Value<T>*> children = {this, &other};
+    auto parent = Value(data_ * other.data_, "*", children);
+    parent.backwards_ = [&parent, &other, this]() {
+      this->grad_ += parent.grad_ * other.data_;
+      other.grad_ += parent.grad_ * this->data_;
+    };
+
+    return parent;
+  }
+
+  Value operator*(T scalar) {
+    return *this * Value(scalar);
+  }
+
+  Value operator-(Value& other) {
+    return *this + (other * -1);
+  }
+
+  Value operator-(T scalar) {
+    return *this - Value(scalar);
+  }
+
+  Value operator^(T exp) {
+    std::vector<Value<T>*> children = {this};
+    auto parent = Value(std::pow(data_, exp), "^", children);
+    parent.backwards_ = [&parent, exp, this]() {
+      this->grad_ += parent.grad_ * exp * std::pow(this->data_, exp - 1);
+    };
+
+    return parent;
+  }
+
+  Value operator/(Value& other) {
+    return *this * (other ^ -1);
+  }
+
+  Value Tanh() {
+    auto parent = Value(std::tanh(data_), "tanh", {this});
+    parent.backwards_ = [&parent, this]() {
+      this->grad_ += parent.grad_ * (1 - std::pow(parent.data_, 2));
+    };
+    return parent;
+  }
+
+  Value ReLU() {
+    auto parent = Value(data_ > 0 ? data_ : 0, "ReLU", {this});
+    parent.backwards_ = [&parent, this]() {
+      this->grad_ += data_ > 0 ? parent.grad_ : 0;
+    };
+    return parent;
+  }
+
+  Value Sigmoid() {
+    auto parent = std::exp(data_) / (1 + std::exp(data_));
+    parent.backwards_ = [&parent, this]() {
+      this->grad_ += parent.grad_ * parent.data_ * (1 - parent.data_);
+    };
+  }
+
+  void Backward() {
+    // Topological sort.
+    std::vector<const Value<T>*> out;
+    std::unordered_set<const Value<T>*> visited;
+    std::queue<const Value<T>*> q;
+    q.push(this);
+
+    while (!q.empty()) {
+      auto node = q.front();
+      q.pop();
+
+      // Already visited. Skip.
+      if (visited.find(node) != visited.end()) {
+        continue;
+      }
+
+      out.push_back(node);
+      visited.insert(node);
+      for (auto child : node->GetChildren()) {
+        q.push(child);
+      }
+    }
+
+    grad_ = 1;
+    for (auto node : out) {
+      if (node->backwards_ != nullptr) {
+        node->backwards_();
+      }
+      std::cout << *node << std::endl;
+    }
+  }
+
+  std::vector<Value<T>*> GetChildren() const { return children_; }
+
+ private:
+  T data_;
+  T grad_;
+  std::string op_;
+  std::vector<Value<T>*> children_;
+  std::function<void()> backwards_;
+  std::string name_;
+};
+
+template <typename T>
+void Visualize(const Value<T>& val) {
+  std::cout << val;
+  for (auto child : val.GetChildren()) {
+    std::cout << std::endl;
+    Visualize(*child);
+  }
+}
+
+#endif  // VALUE_H

requirements.txt

@@ -0,0 +1,121 @@
+anyio==4.3.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
+attrs==23.2.0
+Babel==2.14.0
+beautifulsoup4==4.12.3
+bleach==6.1.0
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+comm==0.2.2
+contourpy==1.2.1
+cycler==0.12.1
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+executing==2.0.1
+fastjsonschema==2.19.1
+filelock==3.14.0
+fonttools==4.51.0
+fqdn==1.5.1
+fsspec==2024.3.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+idna==3.7
+ipykernel==6.29.4
+ipython==8.24.0
+isoduration==20.11.0
+jedi==0.19.1
+Jinja2==3.1.3
+json5==0.9.25
+jsonpointer==2.4
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+jupyter-events==0.10.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.1
+jupyter_core==5.7.2
+jupyter_server==2.14.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.1.8
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.1
+kiwisolver==1.4.5
+MarkupSafe==2.1.5
+matplotlib==3.8.4
+matplotlib-inline==0.1.7
+mistune==3.0.2
+mpmath==1.3.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.3
+notebook==7.1.3
+notebook_shim==0.2.4
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+overrides==7.7.0
+packaging==24.0
+pandas==2.2.2
+pandocfilters==1.5.1
+parso==0.8.4
+pexpect==4.9.0
+pillow==10.3.0
+platformdirs==4.2.1
+prometheus_client==0.20.0
+prompt-toolkit==3.0.43
+psutil==5.9.8
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pycparser==2.22
+Pygments==2.17.2
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-json-logger==2.0.7
+pytz==2024.1
+PyYAML==6.0.1
+pyzmq==26.0.3
+referencing==0.35.1
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.18.0
+Send2Trash==1.8.3
+six==1.16.0
+sniffio==1.3.1
+soupsieve==2.5
+stack-data==0.6.3
+sympy==1.12
+terminado==0.18.1
+tinycss2==1.3.0
+torch==2.3.0
+torchaudio==2.3.0
+torchvision==0.18.0
+tornado==6.4
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20240316
+typing_extensions==4.11.0
+tzdata==2024.1
+uri-template==1.3.0
+urllib3==2.2.1
+wcwidth==0.2.13
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.8.0