yzma

yzma - Go with your own intelligence

yzma lets you write Go applications that directly integrate llama.cpp for fully local inference using hardware acceleration.

• Run the latest Vision Language Models (VLM) and Large/Small/Tiny Language Models (LLM) on Linux, macOS, or Windows.
• Use any available hardware acceleration such as CUDA, Metal), or Vulkan for maximum performance.
• yzma uses the purego and ffi packages so CGo is not needed.
• Works with the newest llama.cpp releases so you can use the latest features, performance improvements, and bugfixes.

This example uses the SmolLM2-135M-GGUF model:

package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/hybridgroup/yzma/pkg/download"
	"github.com/hybridgroup/yzma/pkg/llama"
)

var (
	modelFile            = "SmolLM2-135M.Q4_K_M.gguf"
	prompt               = "Are you ready to go?"
	libPath              = os.Getenv("YZMA_LIB")
	responseLength int32 = 12
)

func main() {
	llama.Load(libPath)
	llama.LogSet(llama.LogSilent())

	llama.Init()

	model, _ := llama.ModelLoadFromFile(filepath.Join(download.DefaultModelsDir(), modelFile), llama.ModelDefaultParams())
	ctx, _ := llama.InitFromModel(model, llama.ContextDefaultParams())

	vocab := llama.ModelGetVocab(model)

	tokens := llama.Tokenize(vocab, prompt, true, false)

	batch := llama.BatchGetOne(tokens)

	sampler := llama.SamplerChainInit(llama.SamplerChainDefaultParams())
	llama.SamplerChainAdd(sampler, llama.SamplerInitGreedy())

	for pos := int32(0); pos < responseLength; pos += batch.NTokens {
		llama.Decode(ctx, batch)
		token := llama.SamplerSample(sampler, ctx, -1)

		if llama.VocabIsEOG(vocab, token) {
			fmt.Println()
			break
		}

		buf := make([]byte, 36)
		len := llama.TokenToPiece(vocab, token, buf, 0, true)

		fmt.Print(string(buf[:len]))

		batch = llama.BatchGetOne([]llama.Token{token})
	}

	fmt.Println()
}

Install yzma, then download the model using the yzma command line tool:

$ yzma model get -u https://huggingface.co/QuantFactory/SmolLM2-135M-GGUF/resolve/main/SmolLM2-135M.Q4_K_M.gguf

And run the Go program:

$ go run ./examples/hello/


"Yes, I'm ready to go."