Class Indexer

Namespace: Kjarni
Assembly: Kjarni.dll

Document indexer.

public class Indexer : IDisposable

Inheritance

object ← Indexer

Implements

IDisposable

Examples

using var indexer = new Indexer(model: "minilm-l6-v2");
var stats = indexer.Create("my_index", new[] { "docs/" });
Console.WriteLine($"Indexed {stats.DocumentsIndexed} documents");

Constructors

Indexer(string, string, string?, int, int, int, IEnumerable<string>?, IEnumerable<string>?, bool, bool, int, bool)

Create a new Indexer.

public Indexer(string model = "minilm-l6-v2", string device = "cpu", string? cacheDir = null, int chunkSize = 512, int chunkOverlap = 100, int batchSize = 32, IEnumerable<string>? extensions = null, IEnumerable<string>? excludePatterns = null, bool recursive = true, bool includeHidden = false, int maxFileSize = 10485760, bool quiet = false)

Parameters

model string

Embedding model name (default: "minilm-l6-v2")

device string

Compute device - "cpu" or "gpu"

cacheDir string?

Directory to cache downloaded models

chunkSize int

Maximum chunk size in characters

chunkOverlap int

Overlap between chunks in characters. Defaults to a fifth of chunkSize, which is the engine's own rule: overlap exists so a passage straddling a boundary still appears whole in one chunk, and an English sentence runs 75 to 100 characters.

batchSize int

Batch size for embedding operations

extensions IEnumerable<string>?

File extensions to include (null = defaults)

excludePatterns IEnumerable<string>?

Glob patterns to exclude

recursive bool

Recurse into subdirectories

includeHidden bool

Include hidden files

maxFileSize int

Skip files larger than this in bytes

quiet bool

Suppress progress output

Properties

ChunkSize

Get the configured chunk size in characters.

public int ChunkSize { get; }

Property Value

int

Dimension

Get the embedding dimension produced by the model.

public int Dimension { get; }

Property Value

int

ModelName

Get the embedding model name used by this indexer.

public string ModelName { get; }

Property Value

string

Methods

Add(string, IEnumerable<string>, Action<Progress>?, CancelToken?)

Add documents to an existing index.

public int Add(string indexPath, IEnumerable<string> inputs, Action<Progress>? onProgress = null, CancelToken? cancelToken = null)

Parameters

indexPath string

Path to existing index

inputs IEnumerable<string>

List of file or directory paths to add

onProgress Action<Progress>?

Optional progress callback

cancelToken CancelToken?

Optional cancellation token

Returns

int

Number of documents added

Create(string, IEnumerable<string>, bool, Action<Progress>?, CancelToken?)

Create a new index from files/directories.

public IndexStats Create(string indexPath, IEnumerable<string> inputs, bool force = false, Action<Progress>? onProgress = null, CancelToken? cancelToken = null)

Parameters

indexPath string

Path where the index will be created

inputs IEnumerable<string>

List of file or directory paths to index

force bool

Overwrite existing index

onProgress Action<Progress>?

Optional progress callback

cancelToken CancelToken?

Optional cancellation token

Returns

IndexStats

IndexStats with indexing statistics

Delete(string)

Delete an index.

public static void Delete(string indexPath)

Parameters

indexPath string

Path to the index to delete

Dispose()

Releases the native model. The object cannot be used afterwards.

public void Dispose()

~Indexer()

Releases the native model if was not called.

protected ~Indexer()

GetInfo(string)

Get information about an existing index.

public static IndexInfo GetInfo(string indexPath)

Parameters

indexPath string

Path to the index directory

Returns

IndexInfo

IndexInfo with index statistics