add dist files

dist/cli.d.cts

@@ -0,0 +1 @@
+#!/usr/bin/env node

dist/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node

dist/index.d.cts

@@ -0,0 +1,99 @@
+import { b as WARCRecord, A as AsyncIterReader, c as StatusAndHeaders, L as LimitReader, d as StatusAndHeadersParser } from './warcserializer-a7845700.js';
+export { e as AsyncIterReaderOpts, B as BaseAsyncIterReader, g as BaseSerializerBuffer, C as CRLF, f as CRLFCRLF, N as NoConcatInflator, S as SerializerInMemBuffer, l as WARCRecordOpts, a as WARCSerializer, W as WARCSerializerOpts, k as WARCType, j as WARC_1_0, h as WARC_1_1, i as indexOfDoubleCRLF, r as readToDoubleCRLF } from './warcserializer-a7845700.js';
+import { I as IndexerOffsetLength, S as Source, H as HeadersMultiMap, a as StreamResults } from './utils-dfea30f1.js';
+export { R as Request, o as Response, k as SourceReadable, i as SourceReader, n as StreamResult, U as UTFToLatin1, c as appendRequestQuery, b as binaryToString, f as concatChunks, g as getSurt, j as jsonToQueryParams, d as jsonToQueryString, l as latin1ToUTF, m as mfdToQueryParams, e as mfdToQueryString, h as multiValueHeader, p as postToGetUrl, r as rxEscape, s as splitChunk } from './utils-dfea30f1.js';
+import { WritableStreamBuffer } from 'stream-buffers';
+import { Argv } from 'yargs';
+import 'hash-wasm/dist/lib/WASMInterface.js';
+import 'pako';
+
+type WARCParserOpts = {
+    keepHeadersCase?: boolean;
+    parseHttp?: boolean;
+};
+declare class WARCParser implements IndexerOffsetLength {
+    static parse(source: Source, options?: WARCParserOpts): Promise<WARCRecord | null>;
+    static iterRecords(source: Source, options?: WARCParserOpts): AsyncGenerator<WARCRecord, void, unknown>;
+    _offset: number;
+    _warcHeadersLength: number;
+    _headersClass: typeof HeadersMultiMap | typeof Headers;
+    _parseHttp: boolean;
+    _reader: AsyncIterReader;
+    _record: WARCRecord | null;
+    constructor(source: Source, { keepHeadersCase, parseHttp }?: WARCParserOpts);
+    readToNextRecord(): Promise<Uint8Array | null>;
+    _initRecordReader(warcHeaders: StatusAndHeaders): LimitReader;
+    parse(): Promise<WARCRecord | null>;
+    get offset(): number;
+    get recordLength(): number;
+    [Symbol.asyncIterator](): AsyncGenerator<WARCRecord, void, unknown>;
+    _addHttpHeaders(record: WARCRecord, headersParser: StatusAndHeadersParser): Promise<void>;
+}
+
+declare const indexCommandArgs: (yarg: Argv) => Argv<{
+    filenames: string[];
+} & {
+    fields: string[];
+}>;
+type IndexCommandArgs = Awaited<ReturnType<typeof indexCommandArgs>["argv"]>;
+declare const cdxIndexCommandArgs: (yarg: Argv) => Argv<{
+    filenames: string[];
+} & {
+    all: boolean | undefined;
+} & {
+    format: string;
+} & {
+    noSurt: boolean | undefined;
+} & {
+    fields: string[];
+}>;
+type CdxIndexCommandArgs = Awaited<ReturnType<typeof cdxIndexCommandArgs>["argv"]>;
+
+declare const DEFAULT_FIELDS: string[];
+declare abstract class BaseIndexer {
+    opts: Partial<IndexCommandArgs>;
+    fields: string[];
+    reqFields: string[];
+    parseHttp: boolean;
+    constructor(opts?: Partial<IndexCommandArgs>, defaultFields?: string[]);
+    serialize(result: Record<string, any>): string;
+    write(result: Record<string, any>, out: WritableStreamBuffer | NodeJS.WriteStream): void;
+    writeAll(files: StreamResults, out: WritableStreamBuffer | NodeJS.WriteStream): Promise<void>;
+    iterIndex(files: StreamResults): AsyncGenerator<Record<string, any>, void, unknown>;
+    iterRecords(parser: WARCParser, filename: string): AsyncGenerator<Record<string, any>, void, unknown>;
+    filterRecord?(record: WARCRecord): boolean;
+    indexRecord(record: WARCRecord, indexerOffset: IndexerOffsetLength, filename: string): Record<string, any> | null;
+    setField(field: string, record: WARCRecord, result: Record<string, any>): void;
+    getField(field: string, record: WARCRecord): string | number | null | undefined;
+}
+declare class Indexer extends BaseIndexer {
+    constructor(opts?: Partial<IndexCommandArgs>, defaultFields?: string[]);
+}
+declare const DEFAULT_CDX_FIELDS: string[];
+declare const DEFAULT_LEGACY_CDX_FIELDS: string[];
+interface CDXAndRecord {
+    cdx: Record<string, any>;
+    record: WARCRecord;
+    reqRecord: WARCRecord | null;
+}
+declare class CDXIndexer extends Indexer {
+    includeAll: boolean;
+    overrideIndexForAll: boolean;
+    noSurt: boolean;
+    _lastRecord: WARCRecord | null;
+    constructor(opts?: Partial<CdxIndexCommandArgs>);
+    iterRecords(parser: WARCParser, filename: string): AsyncGenerator<Record<string, any>, void, unknown>;
+    filterRecord(record: WARCRecord): boolean;
+    indexRecord(record: WARCRecord | null, indexOffset: IndexerOffsetLength, filename: string): Record<string, any> | null;
+    indexRecordPair(record: WARCRecord, reqRecord: WARCRecord | null, indexOffset: IndexerOffsetLength, filename: string): Record<string, any> | null;
+    serializeCDXJ(result: Record<string, any>): string;
+    serializeCDX11(result: Record<string, any>): string;
+    getField(field: string, record: WARCRecord): string | number | null | undefined;
+}
+declare class CDXAndRecordIndexer extends CDXIndexer {
+    constructor(opts?: Partial<CdxIndexCommandArgs>);
+    indexRecordPair(record: WARCRecord, reqRecord: WARCRecord | null, indexOffset: IndexerOffsetLength, filename: string): CDXAndRecord | null;
+}
+declare function isRequestHeader(header: string): boolean;
+
+export { AsyncIterReader, CDXAndRecord, CDXAndRecordIndexer, CDXIndexer, DEFAULT_CDX_FIELDS, DEFAULT_FIELDS, DEFAULT_LEGACY_CDX_FIELDS, HeadersMultiMap, Indexer, IndexerOffsetLength, LimitReader, Source, StatusAndHeaders, StatusAndHeadersParser, StreamResults, WARCParser, WARCParserOpts, WARCRecord, isRequestHeader };