GGUF llama support

.clang-format

@@ -0,0 +1,32 @@
+---
+Language: Cpp
+BasedOnStyle: LLVM
+IndentWidth: 4
+UseTab: Never
+ColumnLimit: 100
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterCaseLabel: true
+  AfterControlStatement: Always
+  AfterEnum: true
+  AfterFunction: true
+  AfterNamespace: true
+  AfterObjCDeclaration: true
+  AfterStruct: false
+  AfterUnion: false
+  BeforeCatch: true
+  BeforeElse: true
+AllowShortIfStatementsOnASingleLine: Always
+AllowShortLoopsOnASingleLine: true
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortCaseLabelsOnASingleLine: true
+PointerAlignment: Right
+AlignConsecutiveDeclarations: true
+AlignConsecutiveAssignments: true
+IndentCaseLabels: true
+IndentGotoLabels: false
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpaceAfterCStyleCast: false
+SortIncludes: false
+BreakStringLiterals: true

.gitignore

@@ -1,7 +1,8 @@
+build/
 .pio
 .vscode/.browse.c_cpp.db*
 .vscode/c_cpp_properties.json
 .vscode/launch.json
 .vscode/ipch
-emlm.bin
-sdkconfig.*.old
+sdkconfig.*.old
+*.gguf

.vscode/c_cpp_properties.json

@@ -8,11 +8,12 @@
         {
             "name": "PlatformIO",
             "includePath": [
+                "/home/exploit/Projects/EmbedLM/include",
                 "/home/exploit/Projects/EmbedLM/src",
                 "/home/exploit/.platformio/packages/framework-espidf/components/xtensa/esp32s3/include",
                 "/home/exploit/.platformio/packages/framework-espidf/components/xtensa/include",
                 "/home/exploit/.platformio/packages/framework-espidf/components/xtensa/deprecated_include",
-                "/home/exploit/Projects/EmbedLM/.pio/build/rymcu-esp32-s3-devkitc-1/config",
+                "/home/exploit/Projects/EmbedLM/.pio/build/esp32-s3-devkitc-1/config",
                 "/home/exploit/.platformio/packages/framework-espidf/components/newlib/platform_include",
                 "/home/exploit/.platformio/packages/framework-espidf/components/freertos/config/include",
                 "/home/exploit/.platformio/packages/framework-espidf/components/freertos/config/include/freertos",
@@ -189,11 +190,12 @@
             "browse": {
                 "limitSymbolsToIncludedHeaders": true,
                 "path": [
+                    "/home/exploit/Projects/EmbedLM/include",
                     "/home/exploit/Projects/EmbedLM/src",
                     "/home/exploit/.platformio/packages/framework-espidf/components/xtensa/esp32s3/include",
                     "/home/exploit/.platformio/packages/framework-espidf/components/xtensa/include",
                     "/home/exploit/.platformio/packages/framework-espidf/components/xtensa/deprecated_include",
-                    "/home/exploit/Projects/EmbedLM/.pio/build/rymcu-esp32-s3-devkitc-1/config",
+                    "/home/exploit/Projects/EmbedLM/.pio/build/esp32-s3-devkitc-1/config",
                     "/home/exploit/.platformio/packages/framework-espidf/components/newlib/platform_include",
                     "/home/exploit/.platformio/packages/framework-espidf/components/freertos/config/include",
                     "/home/exploit/.platformio/packages/framework-espidf/components/freertos/config/include/freertos",
@@ -389,7 +391,6 @@
             "cppStandard": "gnu++23",
             "compilerPath": "/home/exploit/.platformio/packages/toolchain-xtensa-esp-elf/bin/xtensa-esp32s3-elf-gcc",
             "compilerArgs": [
-                "-mdisable-hardware-atomics",
                 "-mlongcalls",
                 ""
             ]

.vscode/launch.json

@@ -12,8 +12,8 @@
             "type": "platformio-debug",
             "request": "launch",
             "name": "PIO Debug",
-            "executable": "/home/exploit/Projects/EmbedLM/.pio/build/rymcu-esp32-s3-devkitc-1/firmware.elf",
-            "projectEnvName": "rymcu-esp32-s3-devkitc-1",
+            "executable": "/home/exploit/Projects/EmbedLM/.pio/build/esp32-s3-devkitc-1/firmware.elf",
+            "projectEnvName": "esp32-s3-devkitc-1",
             "toolchainBinDir": "/home/exploit/.platformio/packages/toolchain-xtensa-esp-elf/bin",
             "internalConsoleOptions": "openOnSessionStart",
             "preLaunchTask": {
@@ -25,17 +25,17 @@
             "type": "platformio-debug",
             "request": "launch",
             "name": "PIO Debug (skip Pre-Debug)",
-            "executable": "/home/exploit/Projects/EmbedLM/.pio/build/rymcu-esp32-s3-devkitc-1/firmware.elf",
-            "projectEnvName": "rymcu-esp32-s3-devkitc-1",
+            "executable": "/home/exploit/Projects/EmbedLM/.pio/build/esp32-s3-devkitc-1/firmware.elf",
+            "projectEnvName": "esp32-s3-devkitc-1",
             "toolchainBinDir": "/home/exploit/.platformio/packages/toolchain-xtensa-esp-elf/bin",
             "internalConsoleOptions": "openOnSessionStart"
         },
         {
             "type": "platformio-debug",
             "request": "launch",
             "name": "PIO Debug (without uploading)",
-            "executable": "/home/exploit/Projects/EmbedLM/.pio/build/rymcu-esp32-s3-devkitc-1/firmware.elf",
-            "projectEnvName": "rymcu-esp32-s3-devkitc-1",
+            "executable": "/home/exploit/Projects/EmbedLM/.pio/build/esp32-s3-devkitc-1/firmware.elf",
+            "projectEnvName": "esp32-s3-devkitc-1",
             "toolchainBinDir": "/home/exploit/.platformio/packages/toolchain-xtensa-esp-elf/bin",
             "internalConsoleOptions": "openOnSessionStart",
             "loadMode": "manual"

CMakeLists.txt

@@ -1,3 +1,18 @@
 cmake_minimum_required(VERSION 3.16.0)
-include($ENV{IDF_PATH}/tools/cmake/project.cmake)
-project(EmbedLM)
+
+if(DEFINED ENV{IDF_PATH})
+    include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+    project(EmbedLM)
+else()
+    project(EmbedLM C)
+    set(CMAKE_C_STANDARD 11)
+    set(CMAKE_C_STANDARD_REQUIRED ON)
+    option(EMBEDLM_ENABLE_ASAN "Enable AddressSanitizer/LeakSanitizer" OFF)
+    add_executable(embedlm src/gguf.c src/tokenizer.c src/infer.c src/main.c)
+    target_compile_options(embedlm PRIVATE -Wall -Wextra -Wpedantic -O2 -g)
+    if(EMBEDLM_ENABLE_ASAN)
+        target_compile_options(embedlm PRIVATE -fsanitize=address -fno-omit-frame-pointer)
+        target_link_options(embedlm PRIVATE -fsanitize=address)
+    endif()
+    target_link_libraries(embedlm PRIVATE m)
+endif()

README.md

@@ -1,28 +1,45 @@
 # EmbedLM
 
-Optimized inference engine for running quantized GPTNeo models directly on ESP32 microcontrollers. It features INT8 quantization and memory-mapped flash execution to enable LLMs on embedded hardware.
+Optimized GGUF inference engine for running quantized llama models directly on microcontrollers.
 
 ## Quick Start
 
+### ESP32-S3
+
 ```bash
-# Create venv
-python -m venv .venv
+# Build and flash
+pio run -e esp32-s3-devkitc-1 -t upload
+
+# Upload the model
+python ~/.platformio/packages/tool-esptoolpy/esptool.py \
+  --chip esp32s3 \
+  --port /dev/ttyACM1 \
+  --baud 115200 \
+  write_flash 0x400000 model.gguf
+
+# Monitor
+pio device monitor -b 115200
+```
 
-# Activate venv
-source .venv/bin/activate
+### STM32H7
 
-# Install dependencies
-pip install -r requirements.txt
+```bash
+# Build and flash
+pio run -e stm32h743ii-devboard -t upload
 
-# Prepare and quantize the model
-python pack.py
+# Monitor
+pio device monitor -b 115200
+```
 
-# Flash the firmware
-pio run -t upload
+To enable SD-card loading, uncomment this in `platformio.ini`:
 
-# Upload binary using crappy way
-python ~/.platformio/packages/tool-esptoolpy/esptool.py --chip esp32s3 --port /dev/ttyACM1 --baud 115200 write_flash 0x400000 emlm.bin
+```ini
+build_flags =
+    -DEMBEDLM_SDCARD
+```
+
+### Linux
 
-# Monitor output
-pio device monitor
+```bash
+cmake --build build && EMBEDLM_MODEL_PATH=model.gguf ./build/embedlm
 ```