A Deep Learning library / neural net implementation from scratch written in C. I also implemented an autograd engine similar to PyTorch. Done for learning purposes.
I Implemented Tensors as the base data structure (some methods are only 2d like matmul_2d). Also implemented a custom arena memory allocator to use instead of malloc().
Reason behind Arena memory management (specifically for NNs):
On top of the reasons stated in Arena Allocator,
Neural network forward passes generate lots of temporary tensors and other structs. So instead of writing complex functions to free() all the pointers after a backward pass, arena memory management lets you delete everything instantly by setting the offset to 0 (setting the available memory pointer back to the beginning of the allocated memory).
Note: I used arena chaining with linked lists to ensure that addresses remain static and memory blocks are never moved or resized. This prevents realloc() from allocates a completely new and larger block of memory. Instead, arena blocks point to other arena blocks when they fill up in a linked list manner.
Methods implemented:
// tensor.h
typedef struct Tensor;
typedef struct GraphContext;
GraphContext* ctx_create(size_t initial_memory);
void ctx_reset(GraphContext* ctx);
void ctx_destroy(GraphContext* ctx);
Tensor* tensor_create(GraphContext* ctx, int* shape, int rank, bool requires_grad);
Tensor* tensor_create_zeros(GraphContext* ctx, int* shape, int rank, bool requires_grad);
Tensor* tensor_clone(GraphContext* ctx, Tensor* t);
void tensor_zeros(Tensor* t);
void tensor_ones(Tensor* t);
void tensor_rand(Tensor* t, float low, float high);
bool tensor_shape_equal(const Tensor* a, const Tensor* b);
Tensor* tensor_add(GraphContext* ctx, Tensor* a, Tensor* b);
Tensor* tensor_add_scalar(GraphContext* ctx, Tensor* a, float scalar);
Tensor* tensor_mul_elementwise(GraphContext* ctx, Tensor* a, Tensor* b);
Tensor* tensor_matmul_2d(GraphContext* ctx, Tensor* a, Tensor* b);
Tensor* tensor_transpose(GraphContext* ctx, Tensor* t, int dim0, int dim1);
int tensor_argmax(const Tensor* t);
void tensor_print(const Tensor* t);
void tensor_backward(GraphContext* ctx, Tensor* loss);
void tensor_zero_grad(Tensor* t);
// arena.h
typedef struct ArenaBlock;
typedef struct Arena;
void arena_init(Arena *a, size_t size);
void* arena_alloc(Arena *a, size_t size, size_t align);
void* arena_calloc(Arena *a, size_t size, size_t align);
void arena_reset(Arena *a);
void arena_destroy(Arena *a);
// nn.h
typedef struct LinearLayer;
typedef struct LayerNormLayer;
LinearLayer* linear_layer_create(GraphContext* ctx, int input_dim, int output_dim);
void nn_linear_free(LinearLayer* layer);
Tensor* nn_linear_forward(GraphContext* ctx, LinearLayer* layer, Tensor* input);
Tensor* nn_relu_forward(GraphContext* ctx, Tensor* t);
Tensor* nn_gelu_forward(GraphContext* ctx, Tensor* t);
Tensor* nn_softmax_forward(GraphContext* ctx, Tensor* t);
Tensor* nn_layer_norm_forward(GraphContext* ctx, LayerNormLayer* layer, Tensor* input);
Tensor* nn_mse_loss(GraphContext* ctx, Tensor* predictions, Tensor* targets);
Tensor* nn_cross_entropy_loss(GraphContext* ctx, Tensor* predictions, Tensor* targets);
void nn_sgd_update(LinearLayer* layer, float learning_rate);
void nn_rmsprop_update(LinearLayer* layer, float lr, float rho, float epsilon, float weight_decay);
void nn_adamw_update(LinearLayer* layer, float lr, int t, float beta1, float beta2, float epsilon, float weight_decay);