# LogPipeline Source: /tuil/docs/reference/packages/logging/api/log-pipeline Locale: en class exported by @mwillbanks/tuil-logging. {/* Generated by tooling/docs/generate-reference.ts. */} ## class [#class] Public class exported by `@mwillbanks/tuil-logging`. ```ts export class LogPipeline { readonly #parsers: readonly LogParser[]; readonly #enrichers: readonly LogEnricher[]; readonly #redactors: readonly LogRedactor[]; readonly buffer: LogRingBuffer; readonly #history: string[] = []; readonly #saved = new Map(); readonly #deduplicate: boolean; readonly #sampleEvery: number; readonly #maxPerSecond?: number; readonly #queryHistoryLimit: number; readonly #now: () => number; #ingested = 0; #rateWindow = { second: -1, count: 0 }; constructor( options: { readonly parsers?: readonly LogParser[]; readonly enrichers?: readonly LogEnricher[]; readonly redactors?: readonly LogRedactor[]; readonly capacity?: number; readonly deduplicate?: boolean; readonly sampleEvery?: number; readonly maxPerSecond?: number; readonly queryHistoryLimit?: number; readonly now?: () => number; } = {}, ) { this.#parsers = Object.freeze([...(options.parsers ?? builtInLogParsers)]); this.#enrichers = Object.freeze([...(options.enrichers ?? [])]); this.#redactors = Object.freeze([...(options.redactors ?? [])]); this.buffer = new LogRingBuffer(options.capacity); this.#deduplicate = options.deduplicate ?? false; this.#sampleEvery = positiveLogCount( "sampleEvery", options.sampleEvery ?? 1, ); this.#maxPerSecond = options.maxPerSecond === undefined ? undefined : positiveLogCount("maxPerSecond", options.maxPerSecond); this.#queryHistoryLimit = positiveLogCount( "queryHistoryLimit", options.queryHistoryLimit ?? 100, ); this.#now = options.now ?? Date.now; } ingest(input: string, parserId?: string): readonly LogRecord[] { const parser = parserId ? this.#parsers.find((item) => item.id === parserId) : this.#parsers.toSorted( (left, right) => right.detect(input) - left.detect(input), )[0]; if (!parser) { throw new Error(`Log parser "${parserId ?? "detected"}" is unavailable`); } const output = parser.parse(input).map((item) => this.#ingestRecord(item)); return Object.freeze(output); } #ingestRecord(item: LogRecord): LogRecord { const transformed = this.#applyPipeline(item); this.#ingested += 1; if (this.#ingested % this.#sampleEvery !== 0) { this.buffer.recordSampled(); return record({ ...transformed, sampled: true }); } const duplicate = this.#deduplicatedRecord(transformed); if (duplicate) return duplicate; const rateLimited = this.#applyRateLimit(transformed); if (rateLimited.rateLimited) { this.buffer.recordRateLimited(); return rateLimited; } this.buffer.push(rateLimited); return rateLimited; } #applyPipeline(item: LogRecord): LogRecord { let transformed = item; for (const enricher of this.#enrichers) transformed = enricher.enrich(transformed); for (const redactor of this.#redactors) transformed = redactor.redact(transformed); return record({ ...transformed }); } #applyRateLimit(item: LogRecord): LogRecord { const second = Math.floor(this.#now() / 1_000); if (this.#rateWindow.second !== second) { this.#rateWindow = { second, count: 0 }; } this.#rateWindow.count += 1; return this.#maxPerSecond !== undefined && this.#rateWindow.count > this.#maxPerSecond ? record({ ...item, rateLimited: true }) : item; } #deduplicatedRecord(item: LogRecord): LogRecord | undefined { const previous = this.buffer.last(); if (!this.#deduplicate || !previous || !sameLogRecord(previous, item)) return undefined; const duplicate = record({ ...previous, duplicateCount: (previous.duplicateCount ?? 1) + 1, }); this.buffer.replaceLast(duplicate); return duplicate; } query(source: string): readonly LogRecord[] { this.#history.push(source); if (this.#history.length > this.#queryHistoryLimit) this.#history.shift(); return this.filter(source); } filter(source: string): readonly LogRecord[] { const query = compileLogQuery(source); if (query.diagnostics.length > 0) return []; return Object.freeze(this.buffer.records().filter(query.predicate)); } saveSearch(name: string, source: string): void { this.#saved.set(name, source); } savedSearches(): Readonly> { return Object.freeze(Object.fromEntries(this.#saved)); } history(): readonly string[] { return Object.freeze([...this.#history]); } clear(): void { this.#ingested = 0; this.#rateWindow = { second: -1, count: 0 }; this.buffer.clear(); } replay(records: readonly LogRecord[]): void { for (const item of records) this.buffer.push(this.#applyPipeline(item)); } export( records: readonly LogRecord[] = this.buffer.records(), format: "jsonl" | "text" = "jsonl", ): string { if (format === "text") { return records.map((item) => String(item.body)).join("\n"); } return records .map((item) => JSON.stringify(item, (_key, value) => typeof value === "bigint" ? value.toString() : value, ), ) .join("\n"); } } ``` ## Members [#members] | Member | Type | Required | Description | Related types | | --------------------- | ------------------------------------------------------------------------ | -------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | | `#parsers` | `readonly LogParser[]` | Yes | The `#parsers` member uses the `readonly LogParser[]` contract. | [`LogParser`](/tuil/docs/reference/packages/logging/api/log-parser) | | `#enrichers` | `readonly LogEnricher[]` | Yes | The `#enrichers` member uses the `readonly LogEnricher[]` contract. | [`LogEnricher`](/tuil/docs/reference/packages/logging/api/log-enricher) | | `#redactors` | `readonly LogRedactor[]` | Yes | The `#redactors` member uses the `readonly LogRedactor[]` contract. | [`LogRedactor`](/tuil/docs/reference/packages/logging/api/log-redactor) | | `buffer` | `LogRingBuffer` | Yes | The `buffer` member uses the `LogRingBuffer` contract. | — | | `#history` | `string[]` | Yes | The `#history` member uses the `string[]` contract. | — | | `#saved` | `Map` | Yes | The `#saved` member uses the `Map` contract. | — | | `#deduplicate` | `boolean` | Yes | The `#deduplicate` member uses the `boolean` contract. | — | | `#sampleEvery` | `number` | Yes | The `#sampleEvery` member uses the `number` contract. | — | | `#maxPerSecond` | `number \| undefined` | No | The `#maxPerSecond` member uses the `number \| undefined` contract. | — | | `#queryHistoryLimit` | `number` | Yes | The `#queryHistoryLimit` member uses the `number` contract. | — | | `#now` | `() => number` | Yes | The `#now` member uses the `() => number` contract. | — | | `#ingested` | `number` | Yes | The `#ingested` member uses the `number` contract. | — | | `#rateWindow` | `{ second: number; count: number; }` | Yes | The `#rateWindow` member uses the `{ second: number; count: number; }` contract. | — | | `__constructor` | `any` | Yes | The `__constructor` member uses the `any` contract. | — | | `ingest` | `(input: string, parserId?: string) => readonly LogRecord[]` | Yes | The `ingest` member uses the `(input: string, parserId?: string) => readonly LogRecord[]` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | | `#ingestRecord` | `(item: LogRecord) => LogRecord` | Yes | The `#ingestRecord` member uses the `(item: LogRecord) => LogRecord` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | | `#applyPipeline` | `(item: LogRecord) => LogRecord` | Yes | The `#applyPipeline` member uses the `(item: LogRecord) => LogRecord` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | | `#applyRateLimit` | `(item: LogRecord) => LogRecord` | Yes | The `#applyRateLimit` member uses the `(item: LogRecord) => LogRecord` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | | `#deduplicatedRecord` | `(item: LogRecord) => LogRecord \| undefined` | Yes | The `#deduplicatedRecord` member uses the `(item: LogRecord) => LogRecord \| undefined` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | | `query` | `(source: string) => readonly LogRecord[]` | Yes | The `query` member uses the `(source: string) => readonly LogRecord[]` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | | `filter` | `(source: string) => readonly LogRecord[]` | Yes | The `filter` member uses the `(source: string) => readonly LogRecord[]` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | | `saveSearch` | `(name: string, source: string) => void` | Yes | The `saveSearch` member uses the `(name: string, source: string) => void` contract. | — | | `savedSearches` | `() => Readonly>` | Yes | The `savedSearches` member uses the `() => Readonly>` contract. | — | | `history` | `() => readonly string[]` | Yes | The `history` member uses the `() => readonly string[]` contract. | — | | `clear` | `() => void` | Yes | The `clear` member uses the `() => void` contract. | — | | `replay` | `(records: readonly LogRecord[]) => void` | Yes | The `replay` member uses the `(records: readonly LogRecord[]) => void` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | | `export` | `(records?: readonly LogRecord[], format?: "jsonl" \| "text") => string` | Yes | The `export` member uses the `(records?: readonly LogRecord[], format?: "jsonl" \| "text") => string` contract. | [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) | ## Parameters [#parameters] This declaration has no public members. ## Returns [#returns] This declaration does not return a value. ## Throws [#throws] No thrown errors are documented for this declaration. ## Related types [#related-types] * [`LogEnricher`](/tuil/docs/reference/packages/logging/api/log-enricher) * [`LogParser`](/tuil/docs/reference/packages/logging/api/log-parser) * [`LogRecord`](/tuil/docs/reference/packages/logging/api/log-record) * [`LogRedactor`](/tuil/docs/reference/packages/logging/api/log-redactor) ## Source [#source] [View the secondary source reference](https://github.com/mwillbanks/tuil/blob/main/packages/logging/src/index.ts) ## Package [#package] [@mwillbanks/tuil-logging](/tuil/docs/reference/packages/logging)