From c306bee66563c23a08769c7bd6a56f01d5cac138 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 19 Dec 2024 11:08:29 +0100 Subject: [PATCH] fix: adding missing abbreviations files for SentenceSplitter (#8660) * adding missing abbreviations files for SentenceSplitter * fixing tests path --- .pre-commit-config.yaml | 1 + .../preprocessors/sentence_tokenizer.py | 2 +- haystack/data/abbreviations/de.txt | 1097 +++++++++++++++++ haystack/data/abbreviations/en.txt | 975 +++++++++++++++ .../preprocessors/test_sentence_tokenizer.py | 2 +- 5 files changed, 2075 insertions(+), 2 deletions(-) create mode 100644 haystack/data/abbreviations/de.txt create mode 100644 haystack/data/abbreviations/en.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 483ed06b08..306e205e41 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,7 @@ repos: rev: v2.3.0 hooks: - id: codespell + exclude: "haystack/data/abbreviations" args: ["--toml", "pyproject.toml"] additional_dependencies: - tomli diff --git a/haystack/components/preprocessors/sentence_tokenizer.py b/haystack/components/preprocessors/sentence_tokenizer.py index 505126e901..5dd6ad97ee 100644 --- a/haystack/components/preprocessors/sentence_tokenizer.py +++ b/haystack/components/preprocessors/sentence_tokenizer.py @@ -228,7 +228,7 @@ def _read_abbreviations(lang: Language) -> List[str]: :param lang: The language to read the abbreviations for. :returns: List of abbreviations. """ - abbreviations_file = Path(__file__).parent.parent / f"data/abbreviations/{lang}.txt" + abbreviations_file = Path(__file__).parent.parent.parent / f"data/abbreviations/{lang}.txt" if not abbreviations_file.exists(): logger.warning("No abbreviations file found for {language}. Using default abbreviations.", language=lang) return [] diff --git a/haystack/data/abbreviations/de.txt b/haystack/data/abbreviations/de.txt new file mode 100644 index 0000000000..848b07541d --- /dev/null +++ b/haystack/data/abbreviations/de.txt @@ -0,0 +1,1097 @@ +abb +abds +abfr +abg +abgek +abh +abk +abl +ableg +ableu +abm +abn +abr +abs +abschn +abst +abulg +abw +abzgl +accel +add +adhrsg +adj +adr +adv +adyg +aengl +afghan +afr +afrik +afrk +afrs +ags +ahd +aind +akad +akk +akkad +akt +al +alb +alban +alem +alemann +allg +allj +allm +alltagsspr +alphanum +altgr +althochdt +altis +altisländ +altröm +alttest +amer +amerik +amerikan +amhar +amtl +amtlbegr +amtlmitt +amtm +amtsbl +amtsdt +amtsspr +anat +anatom +andalus +ang +ange +angekl +angelsächs +angest +angloamerik +anglofrz +angloind +anh +ank +ankl +anl +anm +annamit +anord +anschl +anschr +antarkt +anthrop +anw +anwbl +anz +aobd +apl +aplprof +apostr +apr +arab +aram +aran +arbrgeg +archäol +arg +argent +arkt +art +aserbaidsch +aslaw +assyr +ast +astron +asächs +attr +auff +aufl +aug +ausdr +ausf +ausg +ausl +aussch +ausschl +ausspr +ausst +austral +außenpol +awar +awest +az +aztek +bab +babl +babyl +bair +bakt +balt +baltoslaw +bankw +banz +baschk +bask +bauf +baupol +bauw +bay +baybgm +baygvbl +bayjmbl +baymabl +bayr +bayvbl +bayärztebl +bbg +bd +bde +bearb +bed +begl +begr +beif +beigel +bej +bek +bekl +bem +ber +berbersprachl +bergb +berger +berl +berlärztebl +berufsbez +bes +besbed +besch +beschl +beschw +beschwger +bestr +betr +betriebswiss +betrverf +bev +bew +bez +bezw +bf +bfn +bft +bg +bgbl +bgm +bhf +bibl +bildl +bildungspol +bildungsspr +biol +bj +bl +blk +bln +bodenk +bpräs +br-dr +br-drs +br-prot +brak-mitt +bras +brem +bremgbl +bret +breton +brn +bruchz +bsd +bsp +bspw +bstbl +bt-dr +bt-drs +btl +btto +bttr +buchw +buddh +bulg +bulgar +bundespol +burjat +burmes +bw +byzant +bz +bzb +bzbl +bzgl +bzw +börsenw +ca +carp +cf +chakass +chald +chant +chem +chilen +chin +chr +christl +chron +co +cresc +dat +ders +dez +dgl +dgr +di +dial +dig +dim +dimin +dingl +dipl +diss +do +dominikan +dor +doz +dr +drchf +drcks +drdes +dres +drhc +drphil +drrernat +drs +drucks +dt +dtl +dto +dtsch +dtzd +dvbl +dz +däbl +dän +ebd +ehem +eidg +eig +eigtl +einf +eingetr +einh +einl +einschl +einstw +einw +eisenb +elektrot +elektrotechn +em +engl +entspr +erb +erf +erg +erl +erm +ersch +erschl +erw +erzb +erzg +erzgeb +eskim +est +estn +etc +etg +etrusk +etw +eur +europ +eust +ev +evang +evtl +ew +ewen +ewenk +exkl +expl +ez +fa +fachspr +fam +feb +fem +ff +fig +finanzmath +finn +finnougr +flgh +flnr +flst +flstk +flstnr +fläm +fn +fnhd +folg +forts +fortstzg +fr +fragm +franz +französ +frdl +frh +frhr +fries +friesl +frl +frnhd +frz +fränk +frühnhd +fsm +ftm +fußn +färö +förml +gabl +gall +galloroman +gart +gaskogn +gbf +geb +gebr +ged +gef +geg +gegr +geh +geisteswissenschaftl +gek +gel +geleg +gem +gemeingerm +gen +geod +geogr +geograf +geograph +geol +geolog +geophys +georg +gep +ger +germ +ges +gesch +geschr +gespr +gest +gesundheitspol +get +gew +gez +gfsch +gft +gg +ggb +ggbfs +ggez +ggf +ggfs +ggs +ggü +ghzg +ghzgt +glchz +gleichbed +gleichz +glz +got +gr +gramm +grammat +graph +grch +grchl +grdb +grdf +grdfl +grdg +grdl +grdr +grds +grdst +griech +grz +grönländ +gvbl +gvnw +gvobl +gyn +gynäk +gz +gäl +hait +halbs +hamb +handw +hbf +hd +hdb +hebr +hess +hethit +hf +hg +hindust +hinr +hins +hinw +hist +hjber +hkl +hl +hmb +hochd +hochspr +hom +hptpl +hpts +hptst +hqu +hr +hrn +hrsg +hs +hubbr +hubr +hw +hyaz +hydr +hydrol +hzm +iallg +iber +ibid +ident +idg +ie +illyr +imkerspr +inc +ind +indef +indekl +indian +indiff +indir +indiv +indog +indogerm +indogerman +indoiran +indon +indones +inf +ing +inh +inkl +inn +innenpol +insb +insbes +int +iron +isl +islam +isländ +it +ital +jabl +jahrh +jakut +jan +jap +japan +jav +jbl +jdn +jem +jg +jh +jhd +jhdt +jhs +jidd +jmbl +jmd +jmdm +jmdn +jmds +jn +journ +jr +jt +jtsd +jugendspr +jugendsprachl +jugoslaw +jul +jun +jur +jvbl +jährl +jüd +kalm +kanad +kap +karib +kastil +katal +kath +kaufm +kaukas +kbgekd +kelt +kfm +kfr +kgr +kindersprachl +kirchenlat +kirchenslaw +kirchl +kirg +kj +kl +klass +klimatol +kol +kom +komm +konf +konj +konv +kop +kopt +korean +kostrsp +kr +kreol +kret +krimgot +kriminaltechn +krit +kroat +krs +ks +ktn +kto +kuban +kurd +kurzw +kw +l +lab +labg +ladin +landespol +landsch +landw +langfr +langj +langob +langobard +lapp +lat +latein +latinis +lautl +lautm +lbd +lbdg +ldkr +led +leg +lett +lfd +lfg +lfm +lfrg +lg +lgbl +lgfr +lgft +lgj +lig +ling +lit +lrh +ls +lst +lt +ltd +luth +luxemb +ma +mabl +mag +malai +marinespr +marx +mask +math +max +mazedon +mbl +mbll +md +mdal +mdj +mdl +mdls +mdt +mech +meckl +med +melanes +mengl +merc +meteorol +meton +mexik +mfr +mfranz +mfrk +mfrz +mfränk +mgl +mglw +mhd +mhdt +mi +mia +mihd +milit +mill +min +mind +mio +mitgl +mitteld +mitteldt +mittelhochdt +mittw +mitw +mlat +mnd +mndd +mniederd +mnl +mo +mod +mong +mr +mrd +mrs +mrz +ms +mschr +msgr +msp +mtl +mundartl +mwst +mz +mär +möbl +mündl +nachf +nachm +nachw +nat +nationalsoz +natsoz +nbfl +nchf +nd +ndd +ndrl +nds +ndssog +ndsvbl +neapolit +neub +neunorweg +neutest +neutr +nhd +niederd +niederdt +niederl +niederld +niem +nl +nlat +nom +nordamerik +nordd +norddt +nordgerm +nordostd +nordostdt +nordwestd +nordwestdt +norw +norweg +nov +nr +ntw +nutzfl +nw +nwvbl +näml +nördl +obb +obd +oberlaus +obers +obersächs +obj +od +offiz +offz +okt +op +org +orig +orth +osk +osman +ostd +ostdt +oz +palästin +pat +pers +peruan +pet +pf +pfd +pfg +philos +phonolog +phryg +phys +phöniz +pkt +pl +plur +polit +poln +polynes +portug +pos +pp +ppa +pr +preuß +prof +prot +prov +provenz +proz +präd +prähist +präs +psych +päd +qmstr +qt +qu +quadr +quar +quart +quat +quäst +rak +rd +rderl +rdnr +reg +regbl +regt +rel +relig +rep +resp +rgbl +rglm +rgstr +rgt +rh +rh-pf +rheinhess +rhet +rhfrk +rhj +rhld +rhs +ri +richtl +rip +rk +rmbl +rn +rotw +rr +rrh +rs +rspr +rumän +russ +rvj +rzp +rätorom +röm +sa +saarl +sachs +sanskr +sbd +sc +scherzh +schles +schr +schriftl +schwed +schwäb +sdp +sek +sem +semit +sen +sep +sept +serb +serbokroat +sg +sibir +singhal +sizilian +skand +slaw +slg +slowak +slowen +sod +sof +sog +sogen +sogl +soldatenspr +solv +somal +sorb +sout +soz +sozialgesch +soziol +spez +sportspr +spr +sprachwiss +spätahd +spätgriech +spätlat +spätmhd +sr +ssp +st +staatl +std +stdl +stellv +stf +str +stud +stuzbl +subsp +subst +sumer +svw +syn +syr +sächs +sächsvbl +südafrik +südd +süddt +südl +südostdt +südwestd +süßw +tabl +taf +tamil +tatar +techn +teilw +tel +telef +terr +tfx +tgl +tgt +thrak +thür +thüring +thürvbl +ti +tib +tirol +tochar +trans +tschech +tschechoslowak +tsd +tungus +turkotat +typogr +tz +tägl +türk +uabs +udgl +ugr +ugs +ukrain +umbr +umstr +unang +unbefl +ungar +ungebr +ungel +univ +unzerbr +urgerm +urk +urkdl +urspr +ursprüngl +urt +usf +ust-idnr +usw +va +var +vbl +vchr +verf +verg +vergl +verh +verkehrspol +vern +vers +verwarch +vfg +vgl +vh +viell +vkbl +vl +vlat +vllt +vlt +vobl +volkst +vollj +vorbem +vors +vs +vsl +vt +vulg +vulgärlat +vwz +vzk +wa +weibl +weißruss +westd +westdt +westf +westfäl +westgerm +westl +wfl +wg +wh +whg +winzerspr +wirtschaftl +wiss +wj +wld +wtb +wwe +wz +xerogr +xerok +xyl +yd +yds +zb +zbsp +zeithist +zf +zi +ziff +zool +zpr +zssg +zssgn +zt +zus +zw +zz +zzgl +zzt +ägypt +änd +öbgbl +ökol +ökon +ökum +österr +östl +übertr +überw +übk +übl +üblw diff --git a/haystack/data/abbreviations/en.txt b/haystack/data/abbreviations/en.txt new file mode 100644 index 0000000000..f23cb43122 --- /dev/null +++ b/haystack/data/abbreviations/en.txt @@ -0,0 +1,975 @@ +abbrev +abd +aberd +aberdeensh +abl +abol +aborig +abp +abr +abridg +abridgem +absol +abst +abstr +acad +acc +accomm +accompl +accs +acct +accts +accus +achievem +add +addit +addr +adj +adjs +adm +admir +admon +admonit +adv +advancem +advb +advert +advoc +advs +advt +advts +aerodynam +aeronaut +aff +afr +agric +agst +al +alch +alg +alleg +allit +alm +alph +alt +amer +analyt +anat +anc +anecd +ang +angl +anglo-ind +anim +ann +anniv +annot +anon +answ +anthrop +anthropol +antiq +aphet +apoc +apol +appl +appl'n +applic +appos +apr +arb +archaeol +archit +argt +arith +arithm +arrangem +arrv +artic +artific +artill +ashm +assemb +assoc +assyriol +astr +astrol +astron +att +attrib +aug +austral +auth +autobiog +autobiogr +ave +ayrsh +bacteriol +bedfordsh +bef +belg +berks +berksh +berw +berwicksh +betw +bibliogr +biochem +biog +biogr +biol +bk +bks +blvd +bord +bp +braz +bros +bur +cal +calc +calend +calif +calligr +camb +cambr +campanol +canad +canterb +capt +cartogr +catal +catech +cath +ceram +cert +certif +cf +ch +chamb +char +charac +chas +chem +chesh +chr +chron +chronol +chrons +cinematogr +circ +cl +classif +climatol +clin +co +col +coll +colloq +com +comm +commandm +commend +commerc +commiss +commonw +communic +comp +compan +compar +compend +compl +compos +conc +conch +concl +concr +conf +confid +confl +confut +congr +congreg +conj +conn +cons +consc +consecr +consid +consol +const +constit +constr +contemp +contempl +contempt +contend +contin +contr +contrib +controv +conv +conversat +convoc +cor +cornw +coron +corp +corr +corresp +counc +courtsh +cpd +craniol +craniom +crim +crit +crt +crts +cryptogr +crystallogr +ct +cumb +cumberld +cumbld +cycl +cytol +dat +dau +deb +dec +declar +ded +def +deliv +dem +demonstr +dep +depred +depredat +dept +derbysh +deriv +derog +descr +deut +devel +devonsh +dict +diffic +dim +dis +discipl +discov +discrim +diss +dist +distemp +distill +distrib +div +divers +dk +doc +doct +domest +dr +drs +durh +dyslog +eccl +eccles +ecclus +ecol +econ +ed +edin +edinb +educ +edw +egypt +egyptol +electr +electro-magn +electro-physiol +elem +eliz +elizab +ellipt +emb +embryol +emph +encl +encycl +eng +engin +englishw +enq +ent +enthus +entom +entomol +enzymol +ep +eph +ephes +epil +episc +epist +epit +equip +erron +esd +esp +ess +essent +establ +esth +etc +ethnol +etym +etymol +euphem +eval +evang +evid +evol +exalt +exc +exch +exec +exerc +exhib +exod +exped +exper +explan +explic +explor +expos +ext +ezek +fab +fam +famil +farew +feb +fem +ff +fifesh +fig +fl +footpr +forfarsh +fortif +fortn +found +fr +fragm +fratern +freq +fri +friendsh +ft +furnit +fut +gal +gard +gastron +gaz +gd +gen +geo +geog +geogr +geol +geom +geomorphol +ger +glac +glasg +glos +gloss +glouc +gloucestersh +gosp +gov +govt +gr +gram +gt +gynaecol +hab +haematol +hag +hampsh +handbk +hants +heb +hebr +hen +herb +heref +herefordsh +hertfordsh +hierogl +hist +histol +hom +horol +hort +hos +hosp +househ +housek +husb +hydraul +hydrol +ichth +icthyol +ideol +idol +illustr +imag +imit +immunol +imp +imperf +impers +impf +impr +improp +inaug +inc +inclos +ind +indef +indic +indir +industr +infin +infl +innoc +inorg +inq +inst +instr +int +intell +interc +interj +interl +internat +interpr +interrog +intr +intrans +intro +introd +inv +invertebr +investig +investm +invoc +ir +irel +iron +irreg +isa +ital +jahrb +jam +jan +jap +jas +jer +joc +josh +jr +jrnl +jrnls +jud +judg +jul +jun +jurisd +jurisdict +jurispr +justif +justific +kgs +kingd +knowl +kpr +lam +lanc +lancash +lancs +lang +langs +lat +lb +ld +lds +lect +leechd +leicest +leicestersh +leics +let +lett +lev +lex +libr +limnol +lincolnsh +lincs +ling +linn +lit +lithogr +lithol +liturg +ll +lond +lt +ltd +macc +mach +mag +magn +mal +managem +manch +manip +manuf +mar +masc +matt +meas +measurem +mech +med +medit +mem +merc +merch +metall +metallif +metallogr +metamorph +metaph +metaphor +meteorol +metrop +mex +mic +mich +microbiol +microsc +midl +mil +milit +min +mineral +misc +miscell +mispr +mon +monum +morphol +mr +mrs +ms +msc +mss +mt +mtg +mts +munic +munif +munim +mus +myst +mythol +nah +narr +narrat +nat +naut +nav +navig +neh +neighb +nerv +neurol +neurosurg +newc +newspr +nom +non-conf +nonce-wd +nonconf +norf +northamptonsh +northants +northumb +northumbld +northumbr +norw +norweg +notts +nov +ns +nucl +num +numism +obad +obed +obj +obl +obs +observ +obstet +obstetr +occas +occup +occurr +oceanogr +oct +offic +okla +ont +ophthalm +ophthalmol +opp +oppress +opt +orac +ord +org +orig +orkn +ornith +ornithol +orthogr +outl +oxf +oxfordsh +oxon +oz +pa +palaeobot +palaeogr +palaeont +palaeontol +paraphr +parasitol +parl +parnass +pathol +peculat +penins +perf +perh +periodontol +pers +persec +personif +perthsh +petrogr +pf +pharm +pharmaceut +pharmacol +phd +phil +philad +philem +philipp +philol +philos +phoen +phonet +phonol +photog +photogr +phr +phrenol +phys +physiogr +physiol +pict +pl +plur +pol +polit +polytechn +porc +poss +posth +postm +ppl +pple +pples +pract +prec +pred +predic +predict +pref +preh +prehist +prep +prerog +pres +presb +preserv +prim +princ +priv +prob +probab +probl +proc +prod +prof +prol +pron +pronunc +prop +propr +pros +prov +provid +provinc +provis +ps +pseudo-arch +pseudo-dial +pseudo-sc +psych +psychoanal +psychoanalyt +psychol +psychopathol +pt +publ +purg +qld +quot +quots +radiol +reas +reb +rec +reclam +recoll +redempt +redupl +ref +refash +refl +refus +refut +reg +regic +regist +regr +rel +relig +reminisc +remonstr +renfrewsh +rep +repr +reprod +reps +rept +repub +res +resid +ret +retrosp +rev +revol +rhet +rich +ross-sh +roxb +roy +rudim +russ +sam +sask +sat +sc +scand +sch +sci +scot +scotl +script +sculpt +seismol +sel +sen +sep +sept +ser +serm +sess +settlem +sev +shakes +shaks +sheph +shetl +shropsh +soc +sociol +som +sonn +sp +spec +specif +specim +spectrosc +spp +sq +sr +ss +st +staffordsh +staffs +stat +statist +ste +str +stratigr +struct +sts +stud +subj +subjunct +subord +subscr +subscript +subseq +subst +suff +superl +suppl +supplic +suppress +surg +surv +sus +syll +symmetr +symp +syst +taxon +techn +technol +tel +telecomm +telegr +teleph +teratol +terminol +terrestr +textbk +theat +theatr +theol +theoret +thermonucl +thes +thess +thur +topogr +tr +trad +trag +trans +transf +transl +transubstant +trav +treas +treatm +trib +trig +trigonom +trop +troub +troubl +tue +typog +typogr +ult +univ +unkn +unnat +unoffic +unstr +usu +utilit +va +vac +valedict +var +varr +vars +vb +vbl +vbs +veg +venet +vertebr +vet +vic +vict +vind +vindic +virg +virol +viz +voc +vocab +vol +vols +voy +vs +vulg +warwicksh +wd +wed +westm +westmld +westmorld +westmrld +wilts +wiltsh +wis +wisd +wk +wkly +wks +wonderf +worc +worcestersh +worcs +writ +yearbk +yng +yorks +yorksh +yr +yrs +zech +zeitschr +zeph +zoogeogr +zool diff --git a/test/components/preprocessors/test_sentence_tokenizer.py b/test/components/preprocessors/test_sentence_tokenizer.py index bf9aab9a9e..f8aaf68fa7 100644 --- a/test/components/preprocessors/test_sentence_tokenizer.py +++ b/test/components/preprocessors/test_sentence_tokenizer.py @@ -54,7 +54,7 @@ def test_read_abbreviations_existing_file(tmp_path, mock_file_content): abbrev_file.write_text(mock_file_content) with patch("haystack.components.preprocessors.sentence_tokenizer.Path") as mock_path: - mock_path.return_value.parent.parent = tmp_path + mock_path.return_value.parent.parent.parent = tmp_path result = SentenceSplitter._read_abbreviations("en") assert result == ["Mr.", "Dr.", "Prof."]