kjarni::generator

Raw text generator for decoder language models.

Generator

pub struct Generator

Raw text generator for decoder language models.

Provides direct access to language model text generation without chat templates, system prompts, or conversation management.

new

pub async fn new(model: &str) -> GeneratorResult<Self>

Creates a Generator with default settings.

builder

pub fn builder(model: &str) -> GeneratorBuilder

Creates a builder for custom configuration.

generate

pub async fn generate(&self, prompt: &str) -> GeneratorResult<String>

Generates text from a prompt.

generate_with_config

pub async fn generate_with_config(&self, prompt: &str, runtime_overrides: &GenerationOverrides) -> GeneratorResult<String>

Generates with custom overrides for this call only.

stream

pub async fn stream(&self, prompt: &str) -> GeneratorResult< std::pin::Pin<Box<dyn Stream<Item = GeneratorResult<GeneratedToken>> + Send>>, >

Streams generated tokens.

stream_with_config

pub async fn stream_with_config(&self, prompt: &str, runtime_overrides: GenerationOverrides) -> GeneratorResult< std::pin::Pin<Box<dyn Stream<Item = GeneratorResult<GeneratedToken>> + Send>>, >

Streams with custom overrides.

stream_text

pub async fn stream_text(&self, prompt: &str) -> GeneratorResult<std::pin::Pin<Box<dyn Stream<Item = GeneratorResult<String>> + Send>>>

Streams as simple strings.

model_type

pub fn model_type(&self) -> ModelType

model_name

pub fn model_name(&self) -> &str

device

pub fn device(&self) -> Device

context_size

pub fn context_size(&self) -> usize

vocab_size

pub fn vocab_size(&self) -> usize

generation_config

pub fn generation_config(&self) -> &ResolvedGenerationConfig

decoder

pub fn decoder(&self) -> &DecoderGenerator

Returns a reference to the underlying decoder generator.

gpu_context

pub fn gpu_context(&self) -> Option<&Arc<WgpuContext>>

GeneratorBuilder

pub struct GeneratorBuilder

Builder for configuring a Generator.

Example

let generator = Generator::builder("gpt2")
    .cpu()
    .temperature(0.8)
    .max_tokens(100)
    .build()
    .await?;

new

pub fn new(model: impl Into<String>) -> Self

Create a new builder for the specified model.

model_path

pub fn model_path(mut self, path: impl Into<PathBuf>) -> Self

Load model from a local path instead of registry.

device

pub fn device(mut self, device: KjarniDevice) -> Self

Set the device for inference.

cpu

pub fn cpu(self) -> Self

Use CPU for inference.

gpu

pub fn gpu(self) -> Self

Use GPU for inference.

with_context

pub fn with_context(mut self, context: Arc<WgpuContext>) -> Self

Provide a pre-created GPU context.

from_preset

pub fn from_preset(preset: &GeneratorPreset) -> Self

Create a builder from a preset.

creative

pub fn creative(mut self) -> Self

Configure for creative generation (high temperature).

precise

pub fn precise(mut self) -> Self

Configure for precise generation (low temperature).

draft

pub fn draft(mut self, name: impl Into<String>, num_tokens: usize) -> Self

Enables speculative decoding with name as the draft model.

The draft must share this model's vocabulary, which in practice means a smaller model of the same family: qwen2.5-0.5b drafting for qwen2.5-1.5b. num_tokens is how many the draft proposes per round; more trades wasted draft work against fewer passes over the target's weights.

prefix_cache

pub fn prefix_cache(mut self, enabled: bool) -> Self

Reuses the KV cache across calls, prefilling only what a new prompt does not already share with the last one.

Pays off whenever successive prompts share a long head: a chat replaying its history, or RAG re-sending the same retrieved passages. On Qwen2.5-0.5B a 2048 token prompt costs 15.96s cold and 0.49s with 1984 of those tokens already cached, for identical logits.

Holds [DEFAULT_PREFIX_CACHE_TOKENS] tokens; use prefix_cache_tokens to choose. Prompts longer than that still work, they simply prefill from scratch.

Off by default, for two reasons. The cache is eagerly allocated f32 KV, so it costs real memory: on Qwen2.5-0.5B, 24 layers of 2 KV heads, roughly 24KB per token, which is 96MB at the default and 768MB if sized to that model's full 32768-token context. And one cache holds one conversation, so generations on a single instance serialise instead of running concurrently. Ignored while speculative decoding is active.

prefix_cache_tokens

pub fn prefix_cache_tokens(mut self, tokens: usize) -> Self

Enables prefix reuse with an explicit capacity in tokens.

Capacity is the longest shared prefix that can be reused, and it is what the memory cost scales with: roughly 24KB per token on Qwen2.5-0.5B, more on models with more layers or KV heads. Clamped to the model's context.

temperature

pub fn temperature(mut self, temp: f32) -> Self

Set the sampling temperature.

max_tokens

pub fn max_tokens(mut self, max: usize) -> Self

Set the maximum number of tokens to generate.

top_p

pub fn top_p(mut self, p: f32) -> Self

Set top-p (nucleus) sampling threshold.

top_k

pub fn top_k(mut self, k: usize) -> Self

Set top-k sampling limit.

min_p

pub fn min_p(mut self, p: f32) -> Self

Set min-p sampling threshold.

repetition_penalty

pub fn repetition_penalty(mut self, penalty: f32) -> Self

Set repetition penalty.

greedy

pub fn greedy(mut self) -> Self

Use greedy decoding (temperature = 0, deterministic).

generation_config

pub fn generation_config(mut self, overrides: GenerationOverrides) -> Self

Set all generation overrides at once.

with_load_config

pub fn with_load_config<F>(mut self, f: F) -> Self where F: FnOnce(LoadConfigBuilder) -> LoadConfigBuilder,

Configure model loading options.

cache_dir

pub fn cache_dir(mut self, path: impl Into<PathBuf>) -> Self

Set the cache directory for model files.

download_policy

pub fn download_policy(mut self, policy: DownloadPolicy) -> Self

Set the download policy.

offline

pub fn offline(mut self) -> Self

Never download models automatically.

quiet

pub fn quiet(mut self) -> Self

Suppress informational output.

allow_warnings

pub fn allow_warnings(mut self) -> Self

Allow suboptimal model choices without warnings.

build

pub async fn build(self) -> GeneratorResult<Generator>

Build the Generator.

DEFAULT_PREFIX_CACHE_TOKENS

pub const DEFAULT_PREFIX_CACHE_TOKENS: usize = 4096

Tokens a prefix cache holds unless the caller picks a size.

Sized against memory rather than the model's context window: KV is eagerly allocated f32, roughly 24KB per token on Qwen2.5-0.5B, so this is about 96MB there while that model's full 32768-token context would be 768MB. Long enough to cover a real conversation or a RAG passage set.

available_models

pub fn available_models() -> Vec<&'static str>

List available generation models.

generate

pub async fn generate(model: &str, prompt: &str) -> GeneratorResult<String>

Generate text with default settings.

is_generator_model

pub fn is_generator_model(model: &str) -> GeneratorResult<()>

Check if a model can be used for generation.

suggested_models

pub fn suggested_models() -> Vec<&'static str>

Get suggested models for generation.

generator::presets

Generator presets for common use cases.

GeneratorPreset

pub struct GeneratorPreset

Preset configuration for generator models.

Fields

• pub name: &'static str,: Preset name for lookup.
• pub model: &'static str,: Model name from registry.
• pub architecture: &'static str,: Model architecture.
• pub recommended_device: KjarniDevice,: Recommended device.
• pub temperature: Option<f32>,: Recommended temperature.
• pub default_max_tokens: usize,: Default max new tokens.
• pub memory_mb: usize,: Approximate memory usage in MB.
• pub description: &'static str,: Human-readable description.

ALL_V1_PRESETS

pub const ALL_V1_PRESETS: &[&GeneratorPreset] = &[ &GENERATOR_FAST_V1, &GENERATOR_BALANCED_V1, &GENERATOR_QUALITY_V1, &GENERATOR_CREATIVE_V1, &GENERATOR_CODE_V1, ]

All V1 presets for iteration.

GENERATOR_BALANCED_V1

pub const GENERATOR_BALANCED_V1: GeneratorPreset = GeneratorPreset

Balanced text generation - good quality with reasonable speed.

GENERATOR_CODE_V1

pub const GENERATOR_CODE_V1: GeneratorPreset = GeneratorPreset

Code generation preset.

GENERATOR_CREATIVE_V1

pub const GENERATOR_CREATIVE_V1: GeneratorPreset = GeneratorPreset

Creative text generation - higher temperature for variety.

GENERATOR_FAST_V1

pub const GENERATOR_FAST_V1: GeneratorPreset = GeneratorPreset

Fast text generation - smallest model, quick responses.

GENERATOR_QUALITY_V1

pub const GENERATOR_QUALITY_V1: GeneratorPreset = GeneratorPreset

Quality text generation - larger model, better outputs.

GeneratorTier

pub enum GeneratorTier

Generator quality tiers for easy selection.

Variants

• Fast,: Fastest, smallest model.
• Balanced,: Balanced speed and quality.
• Quality,: Best quality, larger model.

resolve

pub fn resolve(&self) -> &'static GeneratorPreset

Get the preset for this tier.

preset

pub fn preset(&self) -> &'static GeneratorPreset

Get the preset (alias for resolve).

find_preset

pub fn find_preset(name: &str) -> Option<&'static GeneratorPreset>

Find a preset by name (case-insensitive).

list_presets

pub fn list_presets() -> Vec<&'static str>

List all available preset names.