kjarni::seq2seq

Generic encoder-decoder (seq2seq) text generation.

Seq2SeqGenerator

pub struct Seq2SeqGenerator

Generic text-to-text generator for encoder-decoder models

new

pub async fn new(model: &str) -> Seq2SeqResult<Self>

Create a Seq2SeqGenerator with default settings.

builder

pub fn builder(model: &str) -> Seq2SeqGeneratorBuilder

Create a builder for custom configuration.

generate

pub async fn generate(&self, input: &str) -> Seq2SeqResult<String>

Generate output from input text.

generate_with_config

pub async fn generate_with_config(&self, input: &str, runtime_overrides: &Seq2SeqOverrides) -> Seq2SeqResult<String>

Generate with runtime overrides for this call only

stream

pub async fn stream(&self, input: &str) -> Seq2SeqResult<std::pin::Pin<Box<dyn Stream<Item = Seq2SeqResult<Seq2SeqToken>> + Send>>>

Stream generated tokens.

stream_with_config

pub async fn stream_with_config(&self, input: &str, runtime_overrides: Seq2SeqOverrides) -> Seq2SeqResult<std::pin::Pin<Box<dyn Stream<Item = Seq2SeqResult<Seq2SeqToken>> + Send>>>

Stream with runtime overrides.

stream_text

pub async fn stream_text(&self, input: &str) -> Seq2SeqResult<std::pin::Pin<Box<dyn Stream<Item = Seq2SeqResult<String>> + Send>>>

Stream text

get_model_defaults

pub fn get_model_defaults(&self) -> GenerationConfig

Get the model's default generation config.

model_type

pub fn model_type(&self) -> ModelType

Get the model type.

model_name

pub fn model_name(&self) -> &str

Get the model's CLI name.

device

pub fn device(&self) -> Device

Get the device the model is running on.

context_size

pub fn context_size(&self) -> usize

Get the context window size.

vocab_size

pub fn vocab_size(&self) -> usize

Get the vocabulary size.

gpu_context

pub fn gpu_context(&self) -> Option<&Arc<WgpuContext>>

Get the GPU context if using GPU.

encoder_decoder

pub fn encoder_decoder(&self) -> &EncoderDecoderGenerator

Get a reference to the underlying encoder-decoder generator.

Seq2SeqGeneratorBuilder

pub struct Seq2SeqGeneratorBuilder

Builder for configuring a Seq2SeqGenerator instance

new

pub fn new(model: impl Into<String>) -> Self

Create a new builder for the specified model.

from_path

pub fn from_path(mut self, path: impl Into<PathBuf>) -> Self

Load model from a local path instead of registry.

device

pub fn device(mut self, device: KjarniDevice) -> Self

Set the device for inference.

cpu

pub fn cpu(self) -> Self

Use CPU for inference.

gpu

pub fn gpu(self) -> Self

Use GPU for inference.

with_context

pub fn with_context(mut self, context: Arc<WgpuContext>) -> Self

Provide a pre-created GPU context.

Useful when sharing a context across multiple models.

num_beams

pub fn num_beams(mut self, n: usize) -> Self

Set the number of beams for beam search.

max_length

pub fn max_length(mut self, len: usize) -> Self

Set the maximum output length in tokens.

min_length

pub fn min_length(mut self, len: usize) -> Self

Set the minimum output length in tokens.

length_penalty

pub fn length_penalty(mut self, penalty: f32) -> Self

Set the length penalty for beam search.

early_stopping

pub fn early_stopping(mut self, early: bool) -> Self

Set early stopping for beam search.

no_repeat_ngram_size

pub fn no_repeat_ngram_size(mut self, size: usize) -> Self

Set n-gram blocking size

repetition_penalty

pub fn repetition_penalty(mut self, penalty: f32) -> Self

Set repetition penalty.

greedy

pub fn greedy(mut self) -> Self

Use greedy decoding

high_quality

pub fn high_quality(mut self) -> Self

Use high-quality beam search (6 beams).

with_overrides

pub fn with_overrides(mut self, overrides: Seq2SeqOverrides) -> Self

Apply a preset of overrides.

merge_overrides

pub fn merge_overrides(mut self, overrides: Seq2SeqOverrides) -> Self

Merge additional overrides with existing ones.

with_load_config

pub fn with_load_config<F>(mut self, f: F) -> Self where F: FnOnce(LoadConfigBuilder) -> LoadConfigBuilder,

Configure model loading options.

cache_dir

pub fn cache_dir(mut self, path: impl Into<PathBuf>) -> Self

Set the cache directory for model files.

download_policy

pub fn download_policy(mut self, policy: DownloadPolicy) -> Self

Set the download policy.

offline

pub fn offline(mut self) -> Self

Never download models automatically (offline mode).

quiet

pub fn quiet(mut self) -> Self

Suppress informational output

allow_warnings

pub fn allow_warnings(mut self) -> Self

Allow warnings without printing them.

build

pub async fn build(self) -> Seq2SeqResult<Seq2SeqGenerator>

Build the Seq2SeqGenerator instance

Seq2SeqOverrides

pub struct Seq2SeqOverrides

User-specified overrides for seq2seq generation

Fields

• pub min_length: Option<usize>,: Minimum output length in tokens.
• pub max_length: Option<usize>,: Maximum output length in tokens.
• pub num_beams: Option<usize>,: Number of beams for beam search. 1 = greedy decoding.
• pub length_penalty: Option<f32>,: Length penalty for beam search.
• pub do_sample: Option<bool>,: Whether to use sampling instead of beam search.
• pub early_stopping: Option<bool>,: Stop when num_beams complete sequences are found.
• pub no_repeat_ngram_size: Option<usize>,: Prevent repeating n-grams of this size.
• pub repetition_penalty: Option<f32>,: Penalty for repeating tokens (1.0 = no penalty).

greedy

pub fn greedy() -> Self

Use greedy decoding (fastest, single beam).

is_empty

pub fn is_empty(&self) -> bool

Check if any overrides are set.

merge

pub fn merge(&self, other: &Self) -> Self

Merge with another set of overrides. Values from other take precedence.

Seq2SeqToken

pub struct Seq2SeqToken

A single token from streaming generation.

Fields

• pub text: String,: The decoded text for this token.
• pub id: u32,: The token ID.
• pub is_special: bool,: Whether this is a special token (EOS, PAD, etc.)

Seq2SeqError

pub enum Seq2SeqError

Errors that can occur during seq2seq generation.

Variants

• UnknownModel(String),: Model name not found in registry.
• ModelNotDownloaded(String),: Model exists but hasn't been downloaded.
• DownloadFailed: Download failed.
• LoadFailed: Model loading failed.
• GpuUnavailable,: GPU requested but not available.
• GenerationFailed(#[source] anyhow::Error),: Generation failed during inference.
• IncompatibleModel: Model architecture incompatible with seq2seq.
• InvalidConfig(String),: Invalid configuration.

Seq2SeqResult

pub type Seq2SeqResult<T> = Result<T, Seq2SeqError>

Result type for seq2seq operations.

available_models

pub fn available_models() -> Vec<&'static str>

List all available seq2seq models.

Returns CLI names of models that can be used with Seq2SeqGenerator.

generate

pub async fn generate(model: &str, input: &str) -> Seq2SeqResult<String>

Generate text with default settings.

generate_with_config

pub async fn generate_with_config(model: &str, input: &str, overrides: Seq2SeqOverrides) -> Seq2SeqResult<String>

Generate with custom overrides

is_seq2seq_model

pub fn is_seq2seq_model(model: &str) -> Seq2SeqResult<()>

Check if a model is valid for seq2seq.

Returns Ok(()) if valid, or an error describing why not.

suggested_models

pub fn suggested_models() -> Vec<&'static str>

Get suggested models for seq2seq tasks.