alpaca-cot-en-refine.yaml

# global parameters
project_name: 'Data-Juicer-recipes-alpaca-cot-en'
dataset_path: '/path/to/your/dataset'  # path to your dataset directory or file
export_path: '/path/to/your/dataset.jsonl'

np: 50  # number of subprocess to process your dataset
open_tracer: true

# process schedule
# a list of several process operators with their arguments
process:
  - document_deduplicator: # 104636705
      lowercase: true
      ignore_non_character: true

  - alphanumeric_filter: # 104636381
      tokenization: false
      min_ratio: 0.1
  - character_repetition_filter: # 104630030
      rep_len: 10
      max_ratio: 0.6
  - flagged_words_filter: # 104576967
      lang: en
      tokenization: true
      max_ratio: 0.017
  - maximum_line_length_filter: # 104575811
      min_len: 20
  - text_length_filter: # 104573711
      min_len: 30

  - document_simhash_deduplicator:  # 72855345
      tokenization: space
      window_size: 3
      lowercase: true
      ignore_pattern: '\p{P}'
      num_blocks: 9
      hamming_distance: 7