From 550503371812c230689ae01ce4eb617777f27d37 Mon Sep 17 00:00:00 2001 From: richardbruskiewich Date: Thu, 26 Oct 2023 21:56:35 -0700 Subject: [PATCH] Initial iteration on model normalization --- project/excel/translator_testing_model.xlsx | Bin 15324 -> 16060 bytes .../graphql/translator_testing_model.graphql | 123 ++- .../translator_testing_model.context.jsonld | 43 +- .../jsonld/translator_testing_model.jsonld | 577 ++++++----- .../translator_testing_model.schema.json | 645 +++++++----- project/owl/translator_testing_model.owl.ttl | 339 ++++--- .../protobuf/translator_testing_model.proto | 137 +-- .../shacl/translator_testing_model.shacl.ttl | 949 ++++++++++-------- project/shex/translator_testing_model.shex | 119 +-- .../sqlschema/translator_testing_model.sql | 206 ++-- .../datamodel/pydanticmodel.py | 243 +++-- .../datamodel/pydanticmodel_v2.py | 243 +++-- .../datamodel/translator_testing_model.py | 544 +++++----- .../schema/translator_testing_model.yaml | 272 +++-- 14 files changed, 2549 insertions(+), 1891 deletions(-) diff --git a/project/excel/translator_testing_model.xlsx b/project/excel/translator_testing_model.xlsx index e200159b5c71ae5f304fd50a61c02d302481cdf4..e1f4cb4e5a05f6ad8fe430288d5e748689b89d3f 100644 GIT binary patch delta 8203 zcmeHMcT`i^)=%iYN$*GqF!UAz0YpH0la3&Wf`EkHn+XnGsSy~eND~+kNDv&lG-2o< zB1Mp*6zM1q{e?K+IPiV5zJKPed2cPVR&sOq{q23u-S>BPxo671A(YZk2cLi*0058x z?4DjrqAW*X59R?B45^z~VGj@hpv4~T-ESFsxqEnv-E#MG5DjpJrzE}5Zxg48v~^f% z^c{BNzG5k=3kwTx2hr3DY}LdT#nI*axf@)oaU&6BpFi1gv`XqEA0Z5(%6_RV)7kMZ z141@GHlEZ)j&zD&@)4#RD)N{id?KjW25dk;xb~5Tl77TgT2^|j>UQgnxBSemNeLY} zI)TGvkd*RpK|C?2wPowOxkr@7k|@pYo#VuT zGQOANnD#~nHJ=Oqy0f~0H*qC{duhQl`+D-&iXCS1cB#EXig#+-7g!Cw5ZJ8e%g|>~ z{>cwM+2vn8BOX{+Zg&RI3XnD*>xBZ(_z5!(R3gI1dJ4lz;)F+e>ay!@#PNA}06+vP zfs6-X={_e;b1ZZHP~smvrX# zR^6!vW{b^&T+IMWjf^T8Va+PS2X;=w4D`KO#dM%G;S^Ek#dhpZN_6Uk_I<+FDx zZ&>epzPdpzA8!(IPbXk7rzG~?@)}8aDe>Jg#3ew2f~_5XxTgu4JqLhDQR9|SSm_P` z4tww^Quc&hTHRfQse8{wsJc#l@j_#uIENJL(>25V8B;KeV zQ3+n=PTNh+F}%> zPwpEUvw&Sn?u@tAd1pA+?*)%Jzf2jhMl4E+mwYI6O=X;z3VbUp%QX;XvKb{h;j64Z z!vuyNY1MBTnQ9sBww*NiF8!)iacS94Uu;0Mzr@(Ql+(pDez^{AF56OGe!#-4+BQ#O z<5K=9IFXc9f%nBi9Sh{S2r1JPw`GlLIb9js!9b~Qe&Qu<(c)7Mcor{ zm$afWS=B0Ljh}@Bli6z#mwx@FTBv}rOEm{{RcYa|K$6dd=I4Sm060MOm%$?sXWf+c zs{S-Xx=@f1&2vm*DTpYoG}^ZiIml?T{d{aWBu=MQ4XSB1ZbE5f$<|MxYnuBJfd;KO zkXS>GD%so!rObr&grbR^uaG?ldYkhA(zf)e9r#H43P|XM_h+4D1Fd-{#MW7)_xlto z`+EyR(DzB#qBwd;p$na*+|aUj6wum?#O|56;my(FD^1z%qX_pSjUey5S>0`7^*%J$ zU2=RfawBgkR39NcWPe-RQ;h<_QlJe9L-$h+PwAIw9VU_nYs#uW&U(xzCE((Uj&;hu zdXqbj(yI#Q*>suelEys?2|hy3EKdRGj^J-rKL&XM)RmBs*ciM%7Jj$YwyQ<2fu6{w z17)=(B9l(FVA{Y`og~%wUYk=-@N$Hz$)$~RC2mkHMNr?dPw}(>`Xj*zE>0&f$hTuF zikhzgxCS`<1pKM^4oYIB<#3;wS93YfZ&`H~^F-Kxrbmd2L@F$8X2U?4^WKO7Z$l?6 zcWjyjUx};8cTEzyqXg7VYOPcl36P_gdg|{nPpwk3A`hrlQvNYrD3}iI*_Z{ZU>WP4 z39fi_(8bk|z~b}N;(Z{ZKVfIlvA<5)m57O#RrnxE+{ zwAo0m`o%Yg%I(E=hYDsUC6AIM^f2Bt6R-MmOjq3*gt-u(lDtx$=uu{>s^4yG^{7`T z!mGdfau4L_RhADiPlV}1OqWeou~BlQbHk+W{0YlGb3$0MPGF1fT3eQ>Vf z#m0rh+=4(o9-)E)T7?QGZ#2`6mRLG_{4Y-z5#k(I&>Xw7ur3 z+4E?mE44RX6-7x+{5_w1&nN$F`DEIR`2>nBh&YH& z^9gDbLWpQI)iIbqt&5Td7@X^SR83i9YAQb=!<8#v{t6vwFP)dzEwDU3iXSnuc;)d1 zKdoIV$NfWQFR$hOyBB@{Xdl1oj4UaRo&xFpSelzUOTb~#WSL*VSGCk54W&jeDL>PC z**0r=O~>_0j&B|8k>i`hA;$F(x{SKiV_zbo$r1!}jBH1vsL%?%bB>&4EB;{qEx`Td z7Y2`ADlg~jDU>~#^OCk)r^)U>MM6TL{1WMo?Rxa79VQgK{NlL+b%}=)t?UnCa(?RW znJx*SW=-N6a9^*eKTMy-P>5^nSTULCbwp^AS@d)cD%QwW4lIAlo>NKgl}d1(yM+!c zn&Ux4+N-+GYXor*1ob5e@RL;I)%O|QWW1VIs(5>sPhFy%p-~=sFY61BE$e1QXmLl{ zmOY8`EXisqb*EnbAftTh9P?%!b@6@P*UkH7J;9iV z_YYso)t53Ih9s-+z&>ONz=dH4JW944@}3HaU)U)THd#53JpIkyh$_?()*KP z8x#p|!S^<_2~0T7H)7?iNl)?{p|lX*G-2k5=6!kD?E(AyJmi}4@@3)0mOx>(R8CRM zC)>!-2afaIs|UWl z@a5>>%}tH26TCAa5+%q80zxqQjJJVQr^WYKt|4S`waY(7HUy4s{~Xy6IJW(BWJBQC z_Ro>c634c`jcn)H1^v#hIiBzA`kO-8d3MQtXV-Uj{aa$!5C<#D0Q$WO`{zQ}e?%2# z@^2y+*Z1=NdwKsimG_r<_%KpjaYeS~%z{a9s2Qu4zDs9M;lf8`K#hRG%0V!p>k)oF?)_ z`yp2)s8Ga^7WCQ;Sp&49ow)G|>I{m0D&OdFynOh*QxJmB>@Wi$~q_ zyH+afrp1-5<}`s>ok=2}kM-F?Mq_`n_R25poiEdAK=Lfu1dMlu+!mTtk1oF}BeS?A z?c0JESLk=8$5DTQYHH4{hVO`hFU(T>o>hLWOsVgEPX|z={6snQK@#F7!@`!7#Xhi| zfcLe4XL|5Z$;GITqfWP%=swu^yff~Kd#0?|Dxl097|z!J7+eIUsOrZn!(5iJZD(bC z$rp#8xW#SXFZ>CB0^HaZy=F`2+al$8cC2Py4-}1x z$t9a9b0|E@335IeF|HuXT*4MS4oC$I!vZ{%CqYx!d*7-KQoQwy7vWF$cC1SATECgF zQh$4H#y2Q{{OP&2)0|Nj%|wOW6B9u(N%A6K@vY)C$l0GQlmIAF8qSL*($zwuHzK-K zK~)ZHz}lH;db>f!N$#T9PBXCWXQ6toJ9=<9*Gea$@5Y4ELS6ijD^TUCv14nm{7O{j z@P2>0-6sa9v9<1%M6edov2WD$hK+yeNZSwsGJ%GZ$-kMKxXN{6K_UB5>@hg@S~lWA zg}?{a^)(;%cY7`4m6kux%Z(iesF%N+%zGGP1k97W&_@`}s9R0LeNA%62q|zYPZ#Ob zp(X^(leth%7!A>_rsZx8=a)>n5!6!??#4?kt1TeA?UNtw$D1IlT`apDk{^DTml~=q z0NoCs*I?tE%P$lU>rBU2%2f(YMp};Jzcim|(9QMQERhyiPq|Y7 z02ctLDi{MMR7wTNw^u=WuyQn4(TtdUR;gXDh+I0BD%vpA0pii)|FI2;T9Ss5p7!;) zeMpqB420|SEZsVbmxc%c$l^b>`R#)Q4xqwhE^?n@;ud|WWw47{fe-*-{ssXc!)QoQ zp7M;SaTyvM@Q?jE;`6{@G8B(21Bw$uivxb!7da0!ro;j5e+T}x#g8;NVCnC`Z*MEk zTWq_418zcSaS={imJLsW5i0`YX7bm1%}hAG5ZSL&{%fEJ>nTtI*TSB2`kxw+^V7;? z$C)RA{%`XvXM9Y`h4Zn&{|gM@!2wgDUvG2^%7(Ydixa*5lZX|J6M^Kuj++825TnI% zstfx)JFcDp4(KBH&0LD$sfb?Yw4FM4r>N86x`c2ld%vqt%Cq4eh~PxZ@?WiF%CjM2 z#Q=Z+xR|$3Al$+G|1f1+hQe66g=Xu0{?j7OV@4 Lu!{)*J-zrZH!FPQ delta 6365 zcmeHLXIPV2(@sJ!p@tx#Nl7R{LRFN|L6IH-=_0*F>1`E5Q4lEtp@<+|DT0DjL1{ry zM4B3^ARqk&)}T?$GF005u` zOh#KIg3EBE!AuFIm$^Wid=3NvSV^O!r-PA?rsO zl1o4+vQhyPzS7;&d7`~8jrQvkgJplFRl|K?_SY2Uy4xZkmap9++(oeOlGMrPbXt#H zkdT6vYei+TErk=};IWcBLsB9PM}-@VLdH2mXO8XwjOlO8em5>|a+rUMPIo3Vo|FrAtp=vb7o55r zapsep1XtQ69{It?(L$!VZ`l^(T<$(_bjB$Hb3Z1IudOvx^e|vUrQ+qDPp~efAwba+ zEe7qn38}~rT;_4x-nhC!X;_>KQ*}t~s-O?S~~h2zn*-xuh+WPuo=e0N+= zo#UjaKf_ECu2rz==|$%ev5cD z;X%}j;M6|>uy`<8YcykHLaD1}X;ppn(Y(EdyG=A&Ujp0gHCmH!zkV(qyLssHoTR!w zZ%XOy5+{Z^m%@yi&T&7tz~t>3T@?yk2JD{Acx^BA)|B(Zog)5IpB7PV_tIrIm1SE# z7!u3UN_3Jc_`l7R?NDWaBP%a`THk&@pDbyU`>`1Fo6eLEXBF_C4Fjs#@+Jk1mvgMl zCpCm?q#h-H&iVY?)9jLUT3_a;jp61hs~v9iu%T0`Qg%ZHK4wQ|GN|i(7nZbZa1ocj z6nr%whUs}_@_4DxGpFV)IUKY3@PdZ0v35J{L1*|%fjYArV?HR|EQ%Y&wDVEyUX!iGxvQ6mwxpyPDqyZD8vvq2jnLudLl!RJ3bT7<+`Lm`T{CI`H6 zp!vmM3#pHwkEel1cj8zd7uc;bAFy*|Ug}A2a;pgF5N2LtHR?T!K)Vw}DnV5>* z1?_Wav&oscHV4vGA1Ar+clVXub6@{cBL<{PzLO%(Uyjwbma9$~q-vq)uc8amw3w^u zOR5v*yVBJ0;hy%0F?J@?VN2L8>JmE)pmV9zTYw#mQB0MAfXz9m2g4^b=pUQ572kJq zRVsQQVj<)iNl`Y8ds;pTWjb z2=ULtwihto?y&Tk&uZU_$Fn=E?gEVE`0EQ0?975i&bmz0|nqDrrwWSND6!JB+ zbvP|oT~)(AQ(eeAi9>#|9vF@G30KV)r_4#vd8cM?0lhH%UR?im+R{spAe%F)J@;~N zJ@5jmG)1BWAZKd(mUrczDf4u_;bQ0iX1Zb7kZ_>L{JcV)%hq=HGaL26Req;9P1mjR z31(Vl(0ZeKfdiZEm-WJBmHn3Z5lE>26iMX%jU@j@lK+)R@>d{~|JQy0b>IIL_x%v= zYT`!%!s8S_f$%UH2#tRQ!m|z$GzD(#OPST@xerC_zQe3v$Xo=e&l#Va;n`a6uhM8@ z00ZE*#JYs&$f8?1T-T^e>4mT@pLMT{iQ5kh$W-x}4B<-?%_cv`Rp}@u7KUJ_mLY47 zRS_wMtz(u`P5gU67_C5sBa(oSnGA&7IRmM5^PRp~r>wlx@^#TB>2=w9X*9>L2fggQ zq7uNCF(kN{w`}cf&m$ie7H&`fB-`*tn6xxIazg>Z(WE7`d_P2|X?Dvc*oxz9^v9hzojUc6?}Q+khVUE&4Tt_g>x0z@ZkC*(H-(;dDaa9h5{*V zFndktiFN`@gou)AIv5iaLREjNBiCB6o91;2XYX|l*Fp*FW>n-7{BrBu;gWh)E2ci} z*Sj|p(2Fmbn3u1wt>j%Sv(aWW$#g;2PaWDIv@$W-jf_u1qkC7K34u%`WPGe%P|40b zy*>AyjEr=@mnL{+s4#wsld4ii(jG`ox=oMKscw^u6-xQTSU`nJeFXjYjQlT_#G@3c zaQ^?8A<1^c{d0yS+Y$HA8Io*A%DoyBFM@yrrO1}rE28k0qHv&eWvD1Wko-#eTFE8M z03sJyA_yt+1D;kAj>n09H$Kx|#nrg|*4LkuuA0!`1^8jOA1HIqa72pb8xo^zR zQzF;D3G(Bn06Ops9ce^B<9X*9a(6Gy6dYoFqCQNcJQTRPJnCEXiRZ{LdKd(VNINC2 z2xe$=XVW^2%yC@RHp_8!%GZCEpoc$EBhwtjAnqi`$7Gn7M8|>fo@hKu>0;MxM{tj>t$%pg#-QUq8z-X$8}}O=Wjj2hEj^6y~T=Uep|?VFsNq{iBdvtz2E*ah8*m=+UZ+4FqUt>(I6}+o)!WQ zSkZXW%y$DK$JPLru|W!Mu{GKb4}8U<)8is-d*TfZi@F)!NW|WQ)?@C8RK=&1xIY4U z6tD1P>N->f&p9Ry6ZK?P25ZDePrpxV#R+N!pejSo74%>m{Y2gA72(hlFzU@bPSZA4gfm;*lE~9MuFNfUWB#_$F>#kE7tN)Mz!{$nJ9k{Bo&m`mmA%s8by4(hX5#yhm zC}d|nsPx@g$!<(58T;M2Np4IEQj(I+^i>g?EgKli?#UH>-ZkN|OVBD`q|j0+42m_r8hg1d`tg|&iGe5WU8%xvFqcq&nAW*XRdK#( zEm|C>TTp(giuK?*nUgPr^TOC;bZ*jnu&-6^+NE1k*xdn@hp&mM8MQO76*^k+!U>5F zR7Z|fzpnWb?j?tVRuN`7z>5j_A1)4;P<&}BF|^})p?fm;V<_Am=g`2xn_ZW1|1ESazrCdA>7iMUiY2YP zcaA^&By}&&vG+tRN1=;wbSU}~wL?=FJl4D);VtVyEPTLpWEL}?o6+k*jOYZ&uoU)M;Jr`dbVdFDZ(xE?^wUKn+=?&v`E2xj+ z@|5Ignb;dG6I%aS%jHmk9$DM3`VKt+0c+BeA1cw1@nP1+15bG)I9JxK8xq17iQL(l zi#pVBowMV-y;bas({~mt`;>nJ-l^g}NvNr~y2w$zK3wYIG3njeR)Gdn6J1v5nL|%Isc|nnvCP>NaFqZhytGiky z6N6m$#B^#F1Nt(fr$L$w2uoU)L+P$!s>#wAgld(2UJ9V^W;79`$&DDMWwC2zPuG?T zakNseO^o7(CvykaW7HcGJGkMmxPu!p>P?9>Ja7n4a4SZgka&Wp$?`J`|Dh{%a@{;j zuaT}D6E-&+cVQhGS#5q@a^)(kZ`k+i>!5C%vo%Ry8(QW)t1K@f^Raua3+MY=syBKJ-4#tMhYP!_Ae4eY`fDF~9^UMO&b zz_j=ZF=*0JmV!MX*(jspB82bC!|DJoE z2$}mqrT->|v=|HDE6}*0~4Fy)CeupwVhcbkGg5P5kX%E;> sQqmX-1jyj^<$3<1>EACWBLKj@Z-JgHK2e#M2}a_xKn(y4$?bjp4{ssS6aWAK diff --git a/project/graphql/translator_testing_model.graphql b/project/graphql/translator_testing_model.graphql index 481eff9..9e58938 100644 --- a/project/graphql/translator_testing_model.graphql +++ b/project/graphql/translator_testing_model.graphql @@ -1,12 +1,42 @@ +type AcceptanceTestAsset + { + name: String + description: String + inputId: Uriorcurie + inputName: String + predicate: String + outputId: Uriorcurie + outputName: String + expectedOutput: ExpectedOutputEnum + testIssue: TestIssueEnum + semanticSeverity: SemanticSeverityEnum + inV1: Boolean + wellKnown: Boolean + id: Uriorcurie! + tags: [String] + mustPassDate: Date + mustPassEnvironment: TestEnvEnum + query: String + stringEntry: String + direction: DirectionEnum + answerInformalConcept: String + expectedResult: ExpectedResultsEnum + topLevel: Integer + queryNode: NodeEnum + notes: String + } + type AcceptanceTestCase { id: Uriorcurie! name: String description: String - tags: [String] + testEnv: TestEnvEnum + testCaseType: TestCaseTypeEnum + queryType: QueryTypeEnum preconditions: [Precondition] - inputs: [SemanticSmokeTestInput]! - outputs: [SemanticSmokeTestOutput]! + tags: [String] + testAssets: [AcceptanceTestAsset]! } type AcceptanceTestSuite @@ -25,83 +55,75 @@ type BenchmarkTestSuite { } -type Input +type ComplianceTestCase { id: Uriorcurie! name: String description: String + testEnv: TestEnvEnum + testCaseType: TestCaseTypeEnum + queryType: QueryTypeEnum + testAssets: [TestCase]! + preconditions: [Precondition] tags: [String] } -type OneHopTestSuite +type KnowledgeGraphNavigationTestCase { id: Uriorcurie! name: String description: String + testEnv: TestEnvEnum + testCaseType: TestCaseTypeEnum + queryType: QueryTypeEnum + testAssets: [TestCase]! + preconditions: [Precondition] tags: [String] - testMetadata: TestMetadata - testPersona: TestPersonaEnum - testCases: [TestCase] - testCaseSpecification: TestCaseSpecification } -type Output +type OneHopTestCase { id: Uriorcurie! name: String description: String + testEnv: TestEnvEnum + testCaseType: TestCaseTypeEnum + queryType: QueryTypeEnum + testAssets: [TestCase]! + preconditions: [Precondition] tags: [String] } -type Precondition +type OneHopTestSuite { id: Uriorcurie! name: String description: String tags: [String] + testMetadata: TestMetadata + testPersona: TestPersonaEnum + testCases: [TestCase] + testCaseSpecification: TestCaseSpecification } -type QueryAnswerPair - { - name: String - description: String - inputId: Uriorcurie - inputName: String - outputId: Uriorcurie - outputName: String - expectedOutput: ExpectedOutputEnum - testIssue: TestIssueEnum - semanticSeverity: SemanticSeverityEnum - inV1: Boolean - wellKnown: Boolean - id: Uriorcurie! - tags: [String] - } - -type SemanticSmokeTestInput +type Precondition { id: Uriorcurie! name: String description: String tags: [String] - mustPassDate: Date - mustPassEnvironment: EnvironmentEnum - query: String - stringEntry: String - direction: DirectionEnum - answerInformalConcept: String - expectedResult: ExpectedResultsEnum - curie: Curie - topLevel: String - node: String - notes: String } -type SemanticSmokeTestOutput +type QuantitativeTestCase { id: Uriorcurie! name: String description: String + testEnv: TestEnvEnum + testCaseType: TestCaseTypeEnum + queryType: QueryTypeEnum + testAssets: [TestCase]! + preconditions: [Precondition] tags: [String] } @@ -123,6 +145,7 @@ type TestAsset description: String inputId: Uriorcurie inputName: String + predicate: String outputId: Uriorcurie outputName: String expectedOutput: ExpectedOutputEnum @@ -134,24 +157,17 @@ type TestAsset tags: [String] } -type TestAssetCollection - { - id: Uriorcurie! - name: String - description: String - testAssets: [TestCase]! - tags: [String] - } - type TestCase { id: Uriorcurie! name: String description: String - tags: [String] - inputs: [Input] - outputs: [Output] + testEnv: TestEnvEnum + testCaseType: TestCaseTypeEnum + queryType: QueryTypeEnum + testAssets: [TestCase]! preconditions: [Precondition] + tags: [String] } type TestCaseSpecification @@ -168,6 +184,7 @@ type TestEdgeData description: String inputId: Uriorcurie inputName: String + predicate: String outputId: Uriorcurie outputName: String expectedOutput: ExpectedOutputEnum diff --git a/project/jsonld/translator_testing_model.context.jsonld b/project/jsonld/translator_testing_model.context.jsonld index bad4ef6..7061f86 100644 --- a/project/jsonld/translator_testing_model.context.jsonld +++ b/project/jsonld/translator_testing_model.context.jsonld @@ -1,7 +1,7 @@ { "comments": { "description": "Auto generated by LinkML jsonld context generator", - "generation_date": "2023-10-25T17:14:57", + "generation_date": "2023-10-26T21:50:38", "source": "translator_testing_model.yaml" }, "@context": { @@ -46,9 +46,6 @@ "input_id": { "@type": "@id" }, - "inputs": { - "@type": "@id" - }, "must_pass_date": { "@type": "xsd:date" }, @@ -66,12 +63,25 @@ "output_id": { "@type": "@id" }, - "outputs": { - "@type": "@id" - }, "preconditions": { "@type": "@id" }, + "query_node": { + "@context": { + "@vocab": "@null", + "text": "skos:notation", + "description": "skos:prefLabel", + "meaning": "@id" + } + }, + "query_type": { + "@context": { + "@vocab": "@null", + "text": "skos:notation", + "description": "skos:prefLabel", + "meaning": "@id" + } + }, "requires_trapi": { "@type": "xsd:boolean" }, @@ -92,9 +102,25 @@ "test_case_specification": { "@type": "@id" }, + "test_case_type": { + "@context": { + "@vocab": "@null", + "text": "skos:notation", + "description": "skos:prefLabel", + "meaning": "@id" + } + }, "test_cases": { "@type": "@id" }, + "test_env": { + "@context": { + "@vocab": "@null", + "text": "skos:notation", + "description": "skos:prefLabel", + "meaning": "@id" + } + }, "test_issue": { "@context": { "@vocab": "@null", @@ -133,6 +159,9 @@ "meaning": "@id" } }, + "top_level": { + "@type": "xsd:integer" + }, "well_known": { "@type": "xsd:boolean" } diff --git a/project/jsonld/translator_testing_model.jsonld b/project/jsonld/translator_testing_model.jsonld index 0c8a2ec..ba9ab3d 100644 --- a/project/jsonld/translator_testing_model.jsonld +++ b/project/jsonld/translator_testing_model.jsonld @@ -423,12 +423,29 @@ } ] }, + { + "name": "QueryTypeEnum", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/QueryTypeEnum", + "description": "Query", + "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", + "permissible_values": [ + { + "text": "treats", + "aliases": [ + "treats(creative)" + ] + } + ] + }, { "name": "ExpectedOutputEnum", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/ExpectedOutputEnum", "description": "Expected output values for instances of Test Asset or Test Cases(?). (Note: does this Enum overlap with 'ExpectedResultsEnum' below?)", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "permissible_values": [ + { + "text": "Top_Answer" + }, { "text": "Acceptable" }, @@ -439,10 +456,12 @@ "text": "NeverShow" }, { - "text": "1_TopAnswer" + "text": "1_TopAnswer", + "is_a": "Top_Answer" }, { - "text": "4_NeverShow" + "text": "4_NeverShow", + "is_a": "NeverShow" } ] }, @@ -462,6 +481,73 @@ } ] }, + { + "name": "NodeEnum", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/NodeEnum", + "description": "Target node of a Subject-Predicate-Object driven query", + "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", + "permissible_values": [ + { + "text": "subject" + }, + { + "text": "object" + } + ] + }, + { + "name": "TestEnvEnum", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestEnvEnum", + "description": "Testing environments within which a TestSuite is run by a TestRunner scheduled by the TestHarness.", + "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", + "permissible_values": [ + { + "text": "dev", + "description": "Development" + }, + { + "text": "ci", + "description": "Continuous Integration" + }, + { + "text": "test", + "description": "Test" + }, + { + "text": "prod", + "description": "Production" + } + ] + }, + { + "name": "TestCaseTypeEnum", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestCaseTypeEnum", + "description": "Enumerated tags for types of test (generally applied to a TestCase).", + "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", + "permissible_values": [ + { + "text": "acceptance", + "description": "Acceptance test" + }, + { + "text": "quantitative", + "description": "Quantitative test" + }, + { + "text": "compliance", + "description": "Standards compliance test" + }, + { + "text": "kg_navigation", + "description": "Knowledge Graph navigation integration test" + }, + { + "text": "one_hop", + "description": "One Hop navigation test", + "is_a": "kg_navigation" + } + ] + }, { "name": "TestIssueEnum", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestIssueEnum", @@ -472,7 +558,7 @@ }, { "text": "TMKP", - "description": "Text Mining Knowledge Provider generated relationship?" + "description": "'Text Mining Knowledge Provider' generated relationship?" }, { "text": "category too generic" @@ -514,25 +600,6 @@ "text": "decreased" } ] - }, - { - "name": "EnvironmentEnum", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/EnvironmentEnum", - "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "permissible_values": [ - { - "text": "DEV" - }, - { - "text": "CI" - }, - { - "text": "TEST" - }, - { - "text": "PROD" - } - ] } ], "slots": [ @@ -596,10 +663,9 @@ ], "slot_uri": "http://schema.org/additionalType", "multivalued": true, - "owner": "TestAssetCollection", + "owner": "TestEntity", "domain_of": [ - "TestEntity", - "TestAssetCollection" + "TestEntity" ], "range": "string", "@type": "SlotDefinition" @@ -676,6 +742,21 @@ "range": "string", "@type": "SlotDefinition" }, + { + "name": "predicate", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/predicate", + "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", + "aliases": [ + "Query" + ], + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/predicate", + "owner": "TestAsset", + "domain_of": [ + "TestAsset" + ], + "range": "string", + "@type": "SlotDefinition" + }, { "name": "output_id", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/output_id", @@ -810,17 +891,18 @@ { "name": "test_assets", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_assets", - "description": "List of explicitly enumerated Test Assets.", + "description": "List of explicitly enumerated Test Assets. The class attributes of TestAsset would be included in the TestCase versus being referred to by the identifier (curie) of the TestAsset. That is, this would be a list of objects (in JSONSchema serialization) versus a list of strings (where each string is an identifier pointing to another class).", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_assets", "multivalued": true, - "owner": "TestAssetCollection", + "owner": "TestCase", "domain_of": [ - "TestAssetCollection" + "TestCase" ], "range": "TestCase", "required": true, "inlined": true, + "inlined_as_list": true, "@type": "SlotDefinition" }, { @@ -863,9 +945,9 @@ ], "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/must_pass_date", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], "range": "date", "@type": "SlotDefinition" @@ -873,28 +955,28 @@ { "name": "must_pass_environment", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/must_pass_environment", - "description": "The environment in which this test must pass", + "description": "The deployment environment within which this test must pass.", "examples": [ { - "value": "PROD", + "value": "prod", "@type": "Example" }, { - "value": "CI", + "value": "ci", "@type": "Example" }, { - "value": "TEST", + "value": "test", "@type": "Example" } ], "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/must_pass_environment", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], - "range": "EnvironmentEnum", + "range": "TestEnvEnum", "@type": "SlotDefinition" }, { @@ -913,9 +995,9 @@ ], "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/query", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], "range": "string", "@type": "SlotDefinition" @@ -940,9 +1022,9 @@ "disease" ], "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/string_entry", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], "range": "string", "@type": "SlotDefinition" @@ -953,9 +1035,9 @@ "description": "The direction of the expected query result triple", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/direction", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], "range": "DirectionEnum", "@type": "SlotDefinition" @@ -984,9 +1066,9 @@ ], "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/answer_informal_concept", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], "range": "string", "@type": "SlotDefinition" @@ -997,50 +1079,37 @@ "description": "The expected result of the query", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/expected_result", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], "range": "ExpectedResultsEnum", "@type": "SlotDefinition" }, - { - "name": "curie", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/curie", - "description": "The curie of the query", - "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/curie", - "owner": "SemanticSmokeTestInput", - "domain_of": [ - "SemanticSmokeTestInput" - ], - "range": "curie", - "@type": "SlotDefinition" - }, { "name": "top_level", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/top_level", "description": "The answer must return in these many results", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/top_level", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], - "range": "string", + "range": "integer", "@type": "SlotDefinition" }, { - "name": "node", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/node", - "description": "The node of the TRAPI query to replace.", + "name": "query_node", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/query_node", + "description": "The node of the (templated) TRAPI query to replace", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/node", - "owner": "SemanticSmokeTestInput", + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/query_node", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], - "range": "string", + "range": "NodeEnum", "@type": "SlotDefinition" }, { @@ -1049,9 +1118,9 @@ "description": "The notes of the query", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/notes", - "owner": "SemanticSmokeTestInput", + "owner": "AcceptanceTestAsset", "domain_of": [ - "SemanticSmokeTestInput" + "AcceptanceTestAsset" ], "range": "string", "@type": "SlotDefinition" @@ -1066,29 +1135,42 @@ "@type": "SlotDefinition" }, { - "name": "inputs", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/inputs", + "name": "test_env", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_env", + "description": "Deployment environment within which the associated TestSuite is run.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/inputs", - "multivalued": true, + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_env", "owner": "TestCase", "domain_of": [ "TestCase" ], - "range": "Input", + "range": "TestEnvEnum", "@type": "SlotDefinition" }, { - "name": "outputs", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/outputs", + "name": "test_case_type", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_case_type", + "description": "Type of TestCase.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/outputs", - "multivalued": true, + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_case_type", "owner": "TestCase", "domain_of": [ "TestCase" ], - "range": "Output", + "range": "TestCaseTypeEnum", + "@type": "SlotDefinition" + }, + { + "name": "query_type", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/query_type", + "description": "Type of TestCase query.", + "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/query_type", + "owner": "TestCase", + "domain_of": [ + "TestCase" + ], + "range": "QueryTypeEnum", "@type": "SlotDefinition" }, { @@ -1153,63 +1235,90 @@ "@type": "SlotDefinition" }, { - "name": "TestAssetCollection_tags", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/tags", - "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection ", + "name": "TestCase_test_case_type", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_case_type", + "description": "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "mappings": [ - "http://schema.org/additionalType" + "is_a": "test_case_type", + "domain": "TestCase", + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_case_type", + "alias": "test_case_type", + "owner": "TestCase", + "domain_of": [ + "TestCase" ], - "is_a": "tags", - "domain": "TestAssetCollection", - "slot_uri": "http://schema.org/additionalType", + "is_usage_slot": true, + "usage_slot_name": "test_case_type", + "range": "TestCaseTypeEnum", + "@type": "SlotDefinition" + }, + { + "name": "TestCase_test_assets", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_assets", + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", + "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", + "aliases": [ + "Block List" + ], + "is_a": "test_assets", + "domain": "TestCase", + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_assets", "multivalued": true, - "alias": "tags", - "owner": "TestAssetCollection", + "alias": "test_assets", + "owner": "TestCase", "domain_of": [ - "TestAssetCollection" + "TestCase" ], "is_usage_slot": true, - "usage_slot_name": "tags", - "range": "string", + "usage_slot_name": "test_assets", + "range": "TestCase", + "required": true, + "inlined": true, + "inlined_as_list": true, "@type": "SlotDefinition" }, { - "name": "AcceptanceTestCase_inputs", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/inputs", + "name": "TestCase_tags", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/tags", + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "inputs", - "domain": "AcceptanceTestCase", - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/inputs", + "mappings": [ + "http://schema.org/additionalType" + ], + "is_a": "tags", + "domain": "TestCase", + "slot_uri": "http://schema.org/additionalType", "multivalued": true, - "alias": "inputs", - "owner": "AcceptanceTestCase", + "alias": "tags", + "owner": "TestCase", "domain_of": [ - "AcceptanceTestCase" + "TestCase" ], "is_usage_slot": true, - "usage_slot_name": "inputs", - "range": "SemanticSmokeTestInput", - "required": true, + "usage_slot_name": "tags", + "range": "string", "@type": "SlotDefinition" }, { - "name": "AcceptanceTestCase_outputs", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/outputs", + "name": "AcceptanceTestCase_test_assets", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_assets", + "description": "List of explicitly enumerated Test Assets. The class attributes of TestAsset would be included in the TestCase versus being referred to by the identifier (curie) of the TestAsset. That is, this would be a list of objects (in JSONSchema serialization) versus a list of strings (where each string is an identifier pointing to another class).", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "outputs", + "is_a": "TestCase_test_assets", "domain": "AcceptanceTestCase", - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/outputs", + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_assets", "multivalued": true, - "alias": "outputs", + "alias": "test_assets", "owner": "AcceptanceTestCase", "domain_of": [ "AcceptanceTestCase" ], "is_usage_slot": true, - "usage_slot_name": "outputs", - "range": "SemanticSmokeTestOutput", + "usage_slot_name": "test_assets", + "range": "AcceptanceTestAsset", "required": true, + "inlined": true, + "inlined_as_list": true, "@type": "SlotDefinition" } ], @@ -1261,6 +1370,7 @@ "description", "input_id", "input_name", + "predicate", "output_id", "output_name", "expected_output", @@ -1276,29 +1386,9 @@ "@type": "ClassDefinition" }, { - "name": "TestAssetCollection", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestAssetCollection", - "description": "Represents an ad hoc list of Test Assets.", - "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "aliases": [ - "BlockList" - ], - "is_a": "TestEntity", - "slots": [ - "id", - "name", - "description", - "test_assets", - "TestAssetCollection_tags" - ], - "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestAssetCollection", - "@type": "ClassDefinition" - }, - { - "name": "QueryAnswerPair", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/QueryAnswerPair", - "description": "Represents a QueryAnswerPair specification of a Test Asset", + "name": "AcceptanceTestAsset", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestAsset", + "description": "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "is_a": "TestAsset", "slots": [ @@ -1306,6 +1396,7 @@ "description", "input_id", "input_name", + "predicate", "output_id", "output_name", "expected_output", @@ -1314,16 +1405,26 @@ "in_v1", "well_known", "TestAsset_id", - "TestAsset_tags" + "TestAsset_tags", + "must_pass_date", + "must_pass_environment", + "query", + "string_entry", + "direction", + "answer_informal_concept", + "expected_result", + "top_level", + "query_node", + "notes" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/QueryAnswerPair", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestAsset", "@type": "ClassDefinition" }, { "name": "TestEdgeData", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestEdgeData", - "description": "Represents a single Biolink Model compliant instance of an edge that can be used for testing.", + "description": "Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "is_a": "TestAsset", "slots": [ @@ -1331,6 +1432,7 @@ "description", "input_id", "input_name", + "predicate", "output_id", "output_name", "expected_output", @@ -1348,17 +1450,19 @@ { "name": "TestCase", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestCase", - "description": "Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a particular test condition.", + "description": "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "is_a": "TestEntity", "slots": [ "id", "name", "description", - "tags", - "inputs", - "outputs", - "preconditions" + "test_env", + "TestCase_test_case_type", + "query_type", + "TestCase_test_assets", + "preconditions", + "TestCase_tags" ], "slot_usage": {}, "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestCase", @@ -1381,28 +1485,29 @@ "@type": "ClassDefinition" }, { - "name": "TestSuite", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestSuite", - "description": "Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present.", + "name": "AcceptanceTestCase", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestCase", + "description": "See AcceptanceTestAsset above for more details.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestEntity", + "is_a": "TestCase", "slots": [ "id", "name", "description", - "tags", - "test_metadata", - "test_persona", - "test_cases", - "test_case_specification" + "test_env", + "TestCase_test_case_type", + "query_type", + "preconditions", + "TestCase_tags", + "AcceptanceTestCase_test_assets" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestSuite", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestCase", "@type": "ClassDefinition" }, { - "name": "AcceptanceTestCase", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestCase", + "name": "QuantitativeTestCase", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/QuantitativeTestCase", "description": "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "is_a": "TestCase", @@ -1410,69 +1515,86 @@ "id", "name", "description", - "tags", + "test_env", + "TestCase_test_case_type", + "query_type", + "TestCase_test_assets", "preconditions", - "AcceptanceTestCase_inputs", - "AcceptanceTestCase_outputs" + "TestCase_tags" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestCase", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/QuantitativeTestCase", "@type": "ClassDefinition" }, { - "name": "AcceptanceTestSuite", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestSuite", + "name": "ComplianceTestCase", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/ComplianceTestCase", + "description": "TRAPI and Biolink Model standards compliance test", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestSuite", + "is_a": "TestCase", "slots": [ "id", "name", "description", - "tags", - "test_metadata", - "test_persona", - "test_cases", - "test_case_specification" + "test_env", + "TestCase_test_case_type", + "query_type", + "TestCase_test_assets", + "preconditions", + "TestCase_tags" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestSuite", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/ComplianceTestCase", "@type": "ClassDefinition" }, { - "name": "BenchmarkTestSuite", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/BenchmarkTestSuite", - "description": "JsonObj(is_a='TestSuite')", + "name": "KnowledgeGraphNavigationTestCase", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/KnowledgeGraphNavigationTestCase", + "description": "Knowledge Graph navigation integration test", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", + "is_a": "TestCase", + "slots": [ + "id", + "name", + "description", + "test_env", + "TestCase_test_case_type", + "query_type", + "TestCase_test_assets", + "preconditions", + "TestCase_tags" + ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/BenchmarkTestSuite", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/KnowledgeGraphNavigationTestCase", "@type": "ClassDefinition" }, { - "name": "StandardsComplianceTestSuite", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/StandardsComplianceTestSuite", - "description": "Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model.", + "name": "OneHopTestCase", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestCase", + "description": "'One Hop' Knowledge Graph navigation integration test", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestSuite", + "is_a": "KnowledgeGraphNavigationTestCase", "slots": [ "id", "name", "description", - "tags", - "test_metadata", - "test_persona", - "test_cases", - "test_case_specification" + "test_env", + "TestCase_test_case_type", + "query_type", + "TestCase_test_assets", + "preconditions", + "TestCase_tags" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/StandardsComplianceTestSuite", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestCase", "@type": "ClassDefinition" }, { - "name": "OneHopTestSuite", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestSuite", - "description": "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness.", + "name": "TestSuite", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestSuite", + "description": "Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestSuite", + "is_a": "TestEntity", "slots": [ "id", "name", @@ -1484,82 +1606,75 @@ "test_case_specification" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestSuite", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestSuite", "@type": "ClassDefinition" }, { - "name": "Input", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/Input", - "description": "Represents an input to a TestCase", + "name": "AcceptanceTestSuite", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestSuite", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestEntity", + "is_a": "TestSuite", "slots": [ "id", "name", "description", - "tags" + "tags", + "test_metadata", + "test_persona", + "test_cases", + "test_case_specification" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/Input", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/AcceptanceTestSuite", "@type": "ClassDefinition" }, { - "name": "SemanticSmokeTestInput", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/SemanticSmokeTestInput", - "description": "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete.", + "name": "BenchmarkTestSuite", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/BenchmarkTestSuite", + "description": "JsonObj(is_a='TestSuite')", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "Input", - "slots": [ - "id", - "name", - "description", - "tags", - "must_pass_date", - "must_pass_environment", - "query", - "string_entry", - "direction", - "answer_informal_concept", - "expected_result", - "curie", - "top_level", - "node", - "notes" - ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/SemanticSmokeTestInput", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/BenchmarkTestSuite", "@type": "ClassDefinition" }, { - "name": "Output", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/Output", - "description": "Represents an output from a TestCase", + "name": "StandardsComplianceTestSuite", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/StandardsComplianceTestSuite", + "description": "Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestEntity", + "is_a": "TestSuite", "slots": [ "id", "name", "description", - "tags" + "tags", + "test_metadata", + "test_persona", + "test_cases", + "test_case_specification" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/Output", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/StandardsComplianceTestSuite", "@type": "ClassDefinition" }, { - "name": "SemanticSmokeTestOutput", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/SemanticSmokeTestOutput", - "description": "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete.", + "name": "OneHopTestSuite", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestSuite", + "description": "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "Output", + "is_a": "TestSuite", "slots": [ "id", "name", "description", - "tags" + "tags", + "test_metadata", + "test_persona", + "test_cases", + "test_case_specification" ], "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/SemanticSmokeTestOutput", + "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestSuite", "@type": "ClassDefinition" }, { @@ -1581,9 +1696,9 @@ ], "metamodel_version": "1.7.0", "source_file": "translator_testing_model.yaml", - "source_file_date": "2023-10-25T17:13:47", - "source_file_size": 13153, - "generation_date": "2023-10-25T17:14:58", + "source_file_date": "2023-10-26T21:50:36", + "source_file_size": 16629, + "generation_date": "2023-10-26T21:50:39", "@type": "SchemaDefinition", "@context": [ "project/jsonld/translator_testing_model.context.jsonld", diff --git a/project/jsonschema/translator_testing_model.schema.json b/project/jsonschema/translator_testing_model.schema.json index 51c40bf..cdfbf4c 100644 --- a/project/jsonschema/translator_testing_model.schema.json +++ b/project/jsonschema/translator_testing_model.schema.json @@ -1,50 +1,157 @@ { "$defs": { - "AcceptanceTestCase": { + "AcceptanceTestAsset": { "additionalProperties": false, "description": "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete.", "properties": { + "answer_informal_concept": { + "description": "An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0", + "type": "string" + }, "description": { "description": "A human-readable description for a Test Entity", "type": "string" }, + "direction": { + "$ref": "#/$defs/DirectionEnum", + "description": "The direction of the expected query result triple" + }, + "expected_output": { + "$ref": "#/$defs/ExpectedOutputEnum" + }, + "expected_result": { + "$ref": "#/$defs/ExpectedResultsEnum", + "description": "The expected result of the query" + }, "id": { "description": "A unique identifier for a Test Entity", "type": "string" }, - "inputs": { + "in_v1": { + "type": "boolean" + }, + "input_id": { + "type": "string" + }, + "input_name": { + "type": "string" + }, + "must_pass_date": { + "description": "The date by which this test must pass", + "format": "date", + "type": "string" + }, + "must_pass_environment": { + "$ref": "#/$defs/TestEnvEnum", + "description": "The deployment environment within which this test must pass." + }, + "name": { + "description": "A human-readable name for a Test Entity", + "type": "string" + }, + "notes": { + "description": "The notes of the query", + "type": "string" + }, + "output_id": { + "type": "string" + }, + "output_name": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "query": { + "description": "The question a SME would ask", + "type": "string" + }, + "query_node": { + "$ref": "#/$defs/NodeEnum", + "description": "The node of the (templated) TRAPI query to replace" + }, + "semantic_severity": { + "$ref": "#/$defs/SemanticSeverityEnum" + }, + "string_entry": { + "description": "The object of the core triple to be tested", + "type": "string" + }, + "tags": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection ", "items": { "type": "string" }, "type": "array" }, + "test_issue": { + "$ref": "#/$defs/TestIssueEnum" + }, + "top_level": { + "description": "The answer must return in these many results", + "type": "integer" + }, + "well_known": { + "type": "boolean" + } + }, + "required": [ + "id" + ], + "title": "AcceptanceTestAsset", + "type": "object" + }, + "AcceptanceTestCase": { + "additionalProperties": false, + "description": "See AcceptanceTestAsset above for more details.", + "properties": { + "description": { + "description": "A human-readable description for a Test Entity", + "type": "string" + }, + "id": { + "description": "A unique identifier for a Test Entity", + "type": "string" + }, "name": { "description": "A human-readable name for a Test Entity", "type": "string" }, - "outputs": { + "preconditions": { "items": { "type": "string" }, "type": "array" }, - "preconditions": { + "query_type": { + "$ref": "#/$defs/QueryTypeEnum", + "description": "Type of TestCase query." + }, + "tags": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", "items": { "type": "string" }, "type": "array" }, - "tags": { - "description": "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.", + "test_assets": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", "items": { - "type": "string" + "$ref": "#/$defs/AcceptanceTestAsset" }, "type": "array" + }, + "test_case_type": { + "$ref": "#/$defs/TestCaseTypeEnum", + "description": "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" + }, + "test_env": { + "$ref": "#/$defs/TestEnvEnum", + "description": "Deployment environment within which the associated TestSuite is run." } }, "required": [ - "inputs", - "outputs", + "test_assets", "id" ], "title": "AcceptanceTestCase", @@ -79,14 +186,7 @@ }, "test_cases": { "additionalProperties": { - "anyOf": [ - { - "$ref": "#/$defs/TestCase__identifier_optional" - }, - { - "type": "null" - } - ] + "$ref": "#/$defs/TestCase__identifier_optional" }, "description": "List of explicitly enumerated Test Cases.", "type": "object" @@ -112,6 +212,62 @@ "title": "BenchmarkTestSuite", "type": "object" }, + "ComplianceTestCase": { + "additionalProperties": false, + "description": "TRAPI and Biolink Model standards compliance test", + "properties": { + "description": { + "description": "A human-readable description for a Test Entity", + "type": "string" + }, + "id": { + "description": "A unique identifier for a Test Entity", + "type": "string" + }, + "name": { + "description": "A human-readable name for a Test Entity", + "type": "string" + }, + "preconditions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "query_type": { + "$ref": "#/$defs/QueryTypeEnum", + "description": "Type of TestCase query." + }, + "tags": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", + "items": { + "type": "string" + }, + "type": "array" + }, + "test_assets": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", + "items": { + "$ref": "#/$defs/TestCase" + }, + "type": "array" + }, + "test_case_type": { + "$ref": "#/$defs/TestCaseTypeEnum", + "description": "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" + }, + "test_env": { + "$ref": "#/$defs/TestEnvEnum", + "description": "Deployment environment within which the associated TestSuite is run." + } + }, + "required": [ + "test_assets", + "id" + ], + "title": "ComplianceTestCase", + "type": "object" + }, "DirectionEnum": { "description": "", "enum": [ @@ -121,20 +277,10 @@ "title": "DirectionEnum", "type": "string" }, - "EnvironmentEnum": { - "description": "", - "enum": [ - "DEV", - "CI", - "TEST", - "PROD" - ], - "title": "EnvironmentEnum", - "type": "string" - }, "ExpectedOutputEnum": { "description": "Expected output values for instances of Test Asset or Test Cases(?). (Note: does this Enum overlap with 'ExpectedResultsEnum' below?)", "enum": [ + "Top_Answer", "Acceptable", "BadButForgivable", "NeverShow", @@ -153,9 +299,9 @@ "title": "ExpectedResultsEnum", "type": "string" }, - "Input": { + "KnowledgeGraphNavigationTestCase": { "additionalProperties": false, - "description": "Represents an input to a TestCase", + "description": "Knowledge Graph navigation integration test", "properties": { "description": { "description": "A human-readable description for a Test Entity", @@ -169,23 +315,58 @@ "description": "A human-readable name for a Test Entity", "type": "string" }, + "preconditions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "query_type": { + "$ref": "#/$defs/QueryTypeEnum", + "description": "Type of TestCase query." + }, "tags": { - "description": "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.", + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", "items": { "type": "string" }, "type": "array" + }, + "test_assets": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", + "items": { + "$ref": "#/$defs/TestCase" + }, + "type": "array" + }, + "test_case_type": { + "$ref": "#/$defs/TestCaseTypeEnum", + "description": "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" + }, + "test_env": { + "$ref": "#/$defs/TestEnvEnum", + "description": "Deployment environment within which the associated TestSuite is run." } }, "required": [ + "test_assets", "id" ], - "title": "Input", + "title": "KnowledgeGraphNavigationTestCase", "type": "object" }, - "OneHopTestSuite": { + "NodeEnum": { + "description": "Target node of a Subject-Predicate-Object driven query", + "enum": [ + "subject", + "object" + ], + "title": "NodeEnum", + "type": "string" + }, + "OneHopTestCase": { "additionalProperties": false, - "description": "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness.", + "description": "'One Hop' Knowledge Graph navigation integration test", "properties": { "description": { "description": "A human-readable description for a Test Entity", @@ -199,49 +380,49 @@ "description": "A human-readable name for a Test Entity", "type": "string" }, - "tags": { - "description": "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.", + "preconditions": { "items": { "type": "string" }, "type": "array" }, - "test_case_specification": { - "description": "Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)", - "type": "string" + "query_type": { + "$ref": "#/$defs/QueryTypeEnum", + "description": "Type of TestCase query." }, - "test_cases": { - "additionalProperties": { - "anyOf": [ - { - "$ref": "#/$defs/TestCase__identifier_optional" - }, - { - "type": "null" - } - ] + "tags": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", + "items": { + "type": "string" }, - "description": "List of explicitly enumerated Test Cases.", - "type": "object" + "type": "array" }, - "test_metadata": { - "description": "Test metadata describes the external provenance, cross-references and objectives for a given test.", - "type": "string" + "test_assets": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", + "items": { + "$ref": "#/$defs/TestCase" + }, + "type": "array" }, - "test_persona": { - "$ref": "#/$defs/TestPersonaEnum", - "description": "A Test persona describes the user or operational context of a given test." + "test_case_type": { + "$ref": "#/$defs/TestCaseTypeEnum", + "description": "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" + }, + "test_env": { + "$ref": "#/$defs/TestEnvEnum", + "description": "Deployment environment within which the associated TestSuite is run." } }, "required": [ + "test_assets", "id" ], - "title": "OneHopTestSuite", + "title": "OneHopTestCase", "type": "object" }, - "Output": { + "OneHopTestSuite": { "additionalProperties": false, - "description": "Represents an output from a TestCase", + "description": "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness.", "properties": { "description": { "description": "A human-readable description for a Test Entity", @@ -261,12 +442,31 @@ "type": "string" }, "type": "array" + }, + "test_case_specification": { + "description": "Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)", + "type": "string" + }, + "test_cases": { + "additionalProperties": { + "$ref": "#/$defs/TestCase__identifier_optional" + }, + "description": "List of explicitly enumerated Test Cases.", + "type": "object" + }, + "test_metadata": { + "description": "Test metadata describes the external provenance, cross-references and objectives for a given test.", + "type": "string" + }, + "test_persona": { + "$ref": "#/$defs/TestPersonaEnum", + "description": "A Test persona describes the user or operational context of a given test." } }, "required": [ "id" ], - "title": "Output", + "title": "OneHopTestSuite", "type": "object" }, "Precondition": { @@ -299,63 +499,70 @@ "title": "Precondition", "type": "object" }, - "QueryAnswerPair": { + "QuantitativeTestCase": { "additionalProperties": false, - "description": "Represents a QueryAnswerPair specification of a Test Asset", + "description": "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete.", "properties": { "description": { "description": "A human-readable description for a Test Entity", "type": "string" }, - "expected_output": { - "$ref": "#/$defs/ExpectedOutputEnum" - }, "id": { "description": "A unique identifier for a Test Entity", "type": "string" }, - "in_v1": { - "type": "boolean" - }, - "input_id": { - "type": "string" - }, - "input_name": { - "type": "string" - }, "name": { "description": "A human-readable name for a Test Entity", "type": "string" }, - "output_id": { - "type": "string" + "preconditions": { + "items": { + "type": "string" + }, + "type": "array" }, - "output_name": { - "type": "string" - }, - "semantic_severity": { - "$ref": "#/$defs/SemanticSeverityEnum" + "query_type": { + "$ref": "#/$defs/QueryTypeEnum", + "description": "Type of TestCase query." }, "tags": { - "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection ", + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", "items": { "type": "string" }, "type": "array" }, - "test_issue": { - "$ref": "#/$defs/TestIssueEnum" + "test_assets": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", + "items": { + "$ref": "#/$defs/TestCase" + }, + "type": "array" }, - "well_known": { - "type": "boolean" + "test_case_type": { + "$ref": "#/$defs/TestCaseTypeEnum", + "description": "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" + }, + "test_env": { + "$ref": "#/$defs/TestEnvEnum", + "description": "Deployment environment within which the associated TestSuite is run." } }, "required": [ + "test_assets", "id" ], - "title": "QueryAnswerPair", + "title": "QuantitativeTestCase", "type": "object" }, + "QueryTypeEnum": { + "description": "Query", + "enum": [ + "treats" + ], + "title": "QueryTypeEnum", + "type": "string" + }, "SemanticSeverityEnum": { "description": "From Jenn's worksheet, empty or ill defined (needs elaboration)", "enum": [ @@ -366,111 +573,6 @@ "title": "SemanticSeverityEnum", "type": "string" }, - "SemanticSmokeTestInput": { - "additionalProperties": false, - "description": "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete.", - "properties": { - "answer_informal_concept": { - "description": "An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0", - "type": "string" - }, - "curie": { - "description": "The curie of the query", - "type": "string" - }, - "description": { - "description": "A human-readable description for a Test Entity", - "type": "string" - }, - "direction": { - "$ref": "#/$defs/DirectionEnum", - "description": "The direction of the expected query result triple" - }, - "expected_result": { - "$ref": "#/$defs/ExpectedResultsEnum", - "description": "The expected result of the query" - }, - "id": { - "description": "A unique identifier for a Test Entity", - "type": "string" - }, - "must_pass_date": { - "description": "The date by which this test must pass", - "format": "date", - "type": "string" - }, - "must_pass_environment": { - "$ref": "#/$defs/EnvironmentEnum", - "description": "The environment in which this test must pass" - }, - "name": { - "description": "A human-readable name for a Test Entity", - "type": "string" - }, - "node": { - "description": "The node of the TRAPI query to replace.", - "type": "string" - }, - "notes": { - "description": "The notes of the query", - "type": "string" - }, - "query": { - "description": "The question a SME would ask", - "type": "string" - }, - "string_entry": { - "description": "The object of the core triple to be tested", - "type": "string" - }, - "tags": { - "description": "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.", - "items": { - "type": "string" - }, - "type": "array" - }, - "top_level": { - "description": "The answer must return in these many results", - "type": "string" - } - }, - "required": [ - "id" - ], - "title": "SemanticSmokeTestInput", - "type": "object" - }, - "SemanticSmokeTestOutput": { - "additionalProperties": false, - "description": "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete.", - "properties": { - "description": { - "description": "A human-readable description for a Test Entity", - "type": "string" - }, - "id": { - "description": "A unique identifier for a Test Entity", - "type": "string" - }, - "name": { - "description": "A human-readable name for a Test Entity", - "type": "string" - }, - "tags": { - "description": "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "id" - ], - "title": "SemanticSmokeTestOutput", - "type": "object" - }, "StandardsComplianceTestSuite": { "additionalProperties": false, "description": "Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model.", @@ -500,14 +602,7 @@ }, "test_cases": { "additionalProperties": { - "anyOf": [ - { - "$ref": "#/$defs/TestCase__identifier_optional" - }, - { - "type": "null" - } - ] + "$ref": "#/$defs/TestCase__identifier_optional" }, "description": "List of explicitly enumerated Test Cases.", "type": "object" @@ -561,6 +656,9 @@ "output_name": { "type": "string" }, + "predicate": { + "type": "string" + }, "semantic_severity": { "$ref": "#/$defs/SemanticSeverityEnum" }, @@ -584,9 +682,9 @@ "title": "TestAsset", "type": "object" }, - "TestAssetCollection": { + "TestCase": { "additionalProperties": false, - "description": "Represents an ad hoc list of Test Assets.", + "description": "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition.", "properties": { "description": { "description": "A human-readable description for a Test Entity", @@ -600,78 +698,41 @@ "description": "A human-readable name for a Test Entity", "type": "string" }, - "tags": { - "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection ", + "preconditions": { "items": { "type": "string" }, "type": "array" }, - "test_assets": { - "additionalProperties": { - "anyOf": [ - { - "$ref": "#/$defs/TestCase__identifier_optional" - }, - { - "type": "null" - } - ] - }, - "description": "List of explicitly enumerated Test Assets.", - "type": "object" - } - }, - "required": [ - "test_assets", - "id" - ], - "title": "TestAssetCollection", - "type": "object" - }, - "TestCase": { - "additionalProperties": false, - "description": "Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a particular test condition.", - "properties": { - "description": { - "description": "A human-readable description for a Test Entity", - "type": "string" + "query_type": { + "$ref": "#/$defs/QueryTypeEnum", + "description": "Type of TestCase query." }, - "id": { - "description": "A unique identifier for a Test Entity", - "type": "string" - }, - "inputs": { + "tags": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", "items": { "type": "string" }, "type": "array" }, - "name": { - "description": "A human-readable name for a Test Entity", - "type": "string" - }, - "outputs": { + "test_assets": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", "items": { - "type": "string" + "$ref": "#/$defs/TestCase" }, "type": "array" }, - "preconditions": { - "items": { - "type": "string" - }, - "type": "array" + "test_case_type": { + "$ref": "#/$defs/TestCaseTypeEnum", + "description": "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" }, - "tags": { - "description": "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.", - "items": { - "type": "string" - }, - "type": "array" + "test_env": { + "$ref": "#/$defs/TestEnvEnum", + "description": "Deployment environment within which the associated TestSuite is run." } }, "required": [ + "test_assets", "id" ], "title": "TestCase", @@ -707,9 +768,21 @@ "title": "TestCaseSpecification", "type": "object" }, + "TestCaseTypeEnum": { + "description": "Enumerated tags for types of test (generally applied to a TestCase).", + "enum": [ + "acceptance", + "quantitative", + "compliance", + "kg_navigation", + "one_hop" + ], + "title": "TestCaseTypeEnum", + "type": "string" + }, "TestCase__identifier_optional": { "additionalProperties": false, - "description": "Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a particular test condition.", + "description": "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition.", "properties": { "description": { "description": "A human-readable description for a Test Entity", @@ -719,43 +792,52 @@ "description": "A unique identifier for a Test Entity", "type": "string" }, - "inputs": { - "items": { - "type": "string" - }, - "type": "array" - }, "name": { "description": "A human-readable name for a Test Entity", "type": "string" }, - "outputs": { + "preconditions": { "items": { "type": "string" }, "type": "array" }, - "preconditions": { + "query_type": { + "$ref": "#/$defs/QueryTypeEnum", + "description": "Type of TestCase query." + }, + "tags": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", "items": { "type": "string" }, "type": "array" }, - "tags": { - "description": "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.", + "test_assets": { + "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", "items": { - "type": "string" + "$ref": "#/$defs/TestCase" }, "type": "array" + }, + "test_case_type": { + "$ref": "#/$defs/TestCaseTypeEnum", + "description": "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" + }, + "test_env": { + "$ref": "#/$defs/TestEnvEnum", + "description": "Deployment environment within which the associated TestSuite is run." } }, - "required": [], + "required": [ + "test_assets" + ], "title": "TestCase", "type": "object" }, "TestEdgeData": { "additionalProperties": false, - "description": "Represents a single Biolink Model compliant instance of an edge that can be used for testing.", + "description": "Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing.", "properties": { "description": { "description": "A human-readable description for a Test Entity", @@ -787,6 +869,9 @@ "output_name": { "type": "string" }, + "predicate": { + "type": "string" + }, "semantic_severity": { "$ref": "#/$defs/SemanticSeverityEnum" }, @@ -810,6 +895,17 @@ "title": "TestEdgeData", "type": "object" }, + "TestEnvEnum": { + "description": "Testing environments within which a TestSuite is run by a TestRunner scheduled by the TestHarness.", + "enum": [ + "dev", + "ci", + "test", + "prod" + ], + "title": "TestEnvEnum", + "type": "string" + }, "TestIssueEnum": { "description": "", "enum": [ @@ -928,14 +1024,7 @@ }, "test_cases": { "additionalProperties": { - "anyOf": [ - { - "$ref": "#/$defs/TestCase__identifier_optional" - }, - { - "type": "null" - } - ] + "$ref": "#/$defs/TestCase__identifier_optional" }, "description": "List of explicitly enumerated Test Cases.", "type": "object" diff --git a/project/owl/translator_testing_model.owl.ttl b/project/owl/translator_testing_model.owl.ttl index 4626729..24cde4d 100644 --- a/project/owl/translator_testing_model.owl.ttl +++ b/project/owl/translator_testing_model.owl.ttl @@ -13,18 +13,12 @@ ttm:AcceptanceTestCase a owl:Class, rdfs:label "AcceptanceTestCase" ; rdfs:subClassOf [ a owl:Restriction ; owl:minCardinality 1 ; - owl:onProperty ttm:outputs ], + owl:onProperty ttm:test_assets ], [ a owl:Restriction ; - owl:minCardinality 1 ; - owl:onProperty ttm:inputs ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:SemanticSmokeTestInput ; - owl:onProperty ttm:inputs ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:SemanticSmokeTestOutput ; - owl:onProperty ttm:outputs ], + owl:allValuesFrom ttm:AcceptanceTestAsset ; + owl:onProperty ttm:test_assets ], ttm:TestCase ; - skos:definition "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; + skos:definition "See AcceptanceTestAsset above for more details." ; skos:inScheme . ttm:AcceptanceTestSuite a owl:Class, @@ -39,6 +33,20 @@ ttm:BenchmarkTestSuite a owl:Class, skos:definition "JsonObj(is_a='TestSuite')" ; skos:inScheme . +ttm:ComplianceTestCase a owl:Class, + linkml:ClassDefinition ; + rdfs:label "ComplianceTestCase" ; + rdfs:subClassOf ttm:TestCase ; + skos:definition "TRAPI and Biolink Model standards compliance test" ; + skos:inScheme . + +ttm:OneHopTestCase a owl:Class, + linkml:ClassDefinition ; + rdfs:label "OneHopTestCase" ; + rdfs:subClassOf ttm:KnowledgeGraphNavigationTestCase ; + skos:definition "'One Hop' Knowledge Graph navigation integration test" ; + skos:inScheme . + ttm:OneHopTestSuite a owl:Class, linkml:ClassDefinition ; rdfs:label "OneHopTestSuite" ; @@ -46,11 +54,11 @@ ttm:OneHopTestSuite a owl:Class, skos:definition "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness." ; skos:inScheme . -ttm:QueryAnswerPair a owl:Class, +ttm:QuantitativeTestCase a owl:Class, linkml:ClassDefinition ; - rdfs:label "QueryAnswerPair" ; - rdfs:subClassOf ttm:TestAsset ; - skos:definition "Represents a QueryAnswerPair specification of a Test Asset" ; + rdfs:label "QuantitativeTestCase" ; + rdfs:subClassOf ttm:TestCase ; + skos:definition "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; skos:inScheme . ttm:StandardsComplianceTestSuite a owl:Class, @@ -60,25 +68,11 @@ ttm:StandardsComplianceTestSuite a owl:Class, skos:definition "Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model." ; skos:inScheme . -ttm:TestAssetCollection a owl:Class, - linkml:ClassDefinition ; - rdfs:label "TestAssetCollection" ; - rdfs:subClassOf [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:tags ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:tags ], - ttm:TestEntity ; - skos:altLabel "BlockList" ; - skos:definition "Represents an ad hoc list of Test Assets." ; - skos:inScheme . - ttm:TestEdgeData a owl:Class, linkml:ClassDefinition ; rdfs:label "TestEdgeData" ; rdfs:subClassOf ttm:TestAsset ; - skos:definition "Represents a single Biolink Model compliant instance of an edge that can be used for testing." ; + skos:definition "Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing." ; skos:inScheme . ttm:answer_informal_concept a owl:ObjectProperty, @@ -88,13 +82,6 @@ ttm:answer_informal_concept a owl:ObjectProperty, skos:definition "An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0" ; skos:inScheme . -ttm:curie a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "curie" ; - rdfs:range linkml:curie ; - skos:definition "The curie of the query" ; - skos:inScheme . - ttm:description a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "description" ; @@ -152,8 +139,8 @@ ttm:must_pass_date a owl:ObjectProperty, ttm:must_pass_environment a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "must_pass_environment" ; - rdfs:range ttm:EnvironmentEnum ; - skos:definition "The environment in which this test must pass" ; + rdfs:range ttm:TestEnvEnum ; + skos:definition "The deployment environment within which this test must pass." ; skos:inScheme . ttm:name a owl:ObjectProperty, @@ -162,13 +149,6 @@ ttm:name a owl:ObjectProperty, skos:definition "A human-readable name for a Test Entity" ; skos:inScheme . -ttm:node a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "node" ; - rdfs:range linkml:String ; - skos:definition "The node of the TRAPI query to replace." ; - skos:inScheme . - ttm:notes a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "notes" ; @@ -195,6 +175,12 @@ ttm:preconditions a owl:ObjectProperty, rdfs:range ttm:Precondition ; skos:inScheme . +ttm:predicate a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "predicate" ; + skos:altLabel "Query" ; + skos:inScheme . + ttm:query a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "query" ; @@ -202,6 +188,20 @@ ttm:query a owl:ObjectProperty, skos:definition "The question a SME would ask" ; skos:inScheme . +ttm:query_node a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "query_node" ; + rdfs:range ttm:NodeEnum ; + skos:definition "The node of the (templated) TRAPI query to replace" ; + skos:inScheme . + +ttm:query_type a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "query_type" ; + rdfs:range ttm:QueryTypeEnum ; + skos:definition "Type of TestCase query." ; + skos:inScheme . + ttm:requires_trapi a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "requires_trapi" ; @@ -225,13 +225,6 @@ ttm:string_entry a owl:ObjectProperty, skos:definition "The object of the core triple to be tested" ; skos:inScheme . -ttm:test_assets a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "test_assets" ; - rdfs:range ttm:TestCase ; - skos:definition "List of explicitly enumerated Test Assets." ; - skos:inScheme . - ttm:test_case_specification a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "test_case_specification" ; @@ -246,6 +239,13 @@ ttm:test_cases a owl:ObjectProperty, skos:definition "List of explicitly enumerated Test Cases." ; skos:inScheme . +ttm:test_env a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "test_env" ; + rdfs:range ttm:TestEnvEnum ; + skos:definition "Deployment environment within which the associated TestSuite is run." ; + skos:inScheme . + ttm:test_issue a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "test_issue" ; @@ -292,7 +292,7 @@ ttm:test_source a owl:ObjectProperty, ttm:top_level a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "top_level" ; - rdfs:range linkml:String ; + rdfs:range linkml:Integer ; skos:definition "The answer must return in these many results" ; skos:inScheme . @@ -303,27 +303,31 @@ ttm:well_known a owl:ObjectProperty, skos:altLabel "Well Known" ; skos:inScheme . -ttm:Precondition a owl:Class, +ttm:AcceptanceTestAsset a owl:Class, linkml:ClassDefinition ; - rdfs:label "Precondition" ; - rdfs:subClassOf ttm:TestEntity ; - skos:definition "Represents a precondition for a TestCase" ; + rdfs:label "AcceptanceTestAsset" ; + rdfs:subClassOf ttm:TestAsset ; + skos:definition "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; skos:inScheme . -ttm:SemanticSmokeTestInput a owl:Class, +ttm:KnowledgeGraphNavigationTestCase a owl:Class, linkml:ClassDefinition ; - rdfs:label "SemanticSmokeTestInput" ; - rdfs:subClassOf ttm:Input ; - skos:definition "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; + rdfs:label "KnowledgeGraphNavigationTestCase" ; + rdfs:subClassOf ttm:TestCase ; + skos:definition "Knowledge Graph navigation integration test" ; skos:inScheme . -ttm:SemanticSmokeTestOutput a owl:Class, +ttm:Precondition a owl:Class, linkml:ClassDefinition ; - rdfs:label "SemanticSmokeTestOutput" ; - rdfs:subClassOf ttm:Output ; - skos:definition "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; + rdfs:label "Precondition" ; + rdfs:subClassOf ttm:TestEntity ; + skos:definition "Represents a precondition for a TestCase" ; skos:inScheme . + a owl:Class, + ttm:QueryTypeEnum ; + rdfs:label "treats" . + ttm:TestCaseSpecification a owl:Class, linkml:ClassDefinition ; rdfs:label "TestCaseSpecification" ; @@ -346,22 +350,6 @@ ttm:TestMetadata a owl:Class, ttm:DirectionEnum ; rdfs:label "increased" . - a owl:Class, - ttm:EnvironmentEnum ; - rdfs:label "CI" . - - a owl:Class, - ttm:EnvironmentEnum ; - rdfs:label "DEV" . - - a owl:Class, - ttm:EnvironmentEnum ; - rdfs:label "PROD" . - - a owl:Class, - ttm:EnvironmentEnum ; - rdfs:label "TEST" . - a owl:Class, ttm:ExpectedOutputEnum ; rdfs:label "1_TopAnswer" . @@ -382,6 +370,10 @@ ttm:TestMetadata a owl:Class, ttm:ExpectedOutputEnum ; rdfs:label "NeverShow" . + a owl:Class, + ttm:ExpectedOutputEnum ; + rdfs:label "Top_Answer" . + a owl:Class, ttm:ExpectedResultsEnum ; rdfs:label "exclude_bad" . @@ -390,19 +382,17 @@ ttm:TestMetadata a owl:Class, ttm:ExpectedResultsEnum ; rdfs:label "include_good" . -ttm:Input a owl:Class, - linkml:ClassDefinition ; - rdfs:label "Input" ; - rdfs:subClassOf ttm:TestEntity ; - skos:definition "Represents an input to a TestCase" ; - skos:inScheme . + a owl:Class, + ttm:NodeEnum ; + rdfs:label "object" . -ttm:Output a owl:Class, - linkml:ClassDefinition ; - rdfs:label "Output" ; - rdfs:subClassOf ttm:TestEntity ; - skos:definition "Represents an output from a TestCase" ; - skos:inScheme . + a owl:Class, + ttm:NodeEnum ; + rdfs:label "subject" . + +ttm:QueryTypeEnum a owl:Class, + linkml:EnumDefinition ; + linkml:permissible_values . a owl:Class, ttm:SemanticSeverityEnum ; @@ -420,17 +410,17 @@ ttm:TestAsset a owl:Class, linkml:ClassDefinition ; rdfs:label "TestAsset" ; rdfs:subClassOf [ a owl:Restriction ; - owl:allValuesFrom linkml:Uriorcurie ; - owl:onProperty ttm:id ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:tags ], - [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:id ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:id ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:Uriorcurie ; + owl:onProperty ttm:id ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:tags ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:tags ], @@ -438,6 +428,42 @@ ttm:TestAsset a owl:Class, skos:definition "Represents a Test Asset, which is a single specific instance of TestCase-agnostic semantic parameters representing the specification of a Translator test target with inputs and (expected) outputs." ; skos:inScheme . + a owl:Class, + ttm:TestCaseTypeEnum ; + rdfs:label "acceptance" . + + a owl:Class, + ttm:TestCaseTypeEnum ; + rdfs:label "compliance" . + + a owl:Class, + ttm:TestCaseTypeEnum ; + rdfs:label "kg_navigation" . + + a owl:Class, + ttm:TestCaseTypeEnum ; + rdfs:label "one_hop" . + + a owl:Class, + ttm:TestCaseTypeEnum ; + rdfs:label "quantitative" . + + a owl:Class, + ttm:TestEnvEnum ; + rdfs:label "ci" . + + a owl:Class, + ttm:TestEnvEnum ; + rdfs:label "dev" . + + a owl:Class, + ttm:TestEnvEnum ; + rdfs:label "prod" . + + a owl:Class, + ttm:TestEnvEnum ; + rdfs:label "test" . + a owl:Class, ttm:TestIssueEnum ; rdfs:label "TMKP" . @@ -514,18 +540,6 @@ ttm:TestAsset a owl:Class, ttm:TestSourceEnum ; rdfs:label "TranslatorTeam" . -ttm:inputs a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "inputs" ; - rdfs:range ttm:Input ; - skos:inScheme . - -ttm:outputs a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "outputs" ; - rdfs:range ttm:Output ; - skos:inScheme . - ttm:DirectionEnum a owl:Class, linkml:EnumDefinition ; owl:unionOf ( ) ; @@ -538,12 +552,11 @@ ttm:ExpectedResultsEnum a owl:Class, linkml:permissible_values , . -ttm:TestCase a owl:Class, - linkml:ClassDefinition ; - rdfs:label "TestCase" ; - rdfs:subClassOf ttm:TestEntity ; - skos:definition "Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a particular test condition." ; - skos:inScheme . +ttm:NodeEnum a owl:Class, + linkml:EnumDefinition ; + owl:unionOf ( ) ; + linkml:permissible_values , + . ttm:TestSuite a owl:Class, linkml:ClassDefinition ; @@ -559,6 +572,13 @@ ttm:id a owl:ObjectProperty, skos:definition "A unique identifier for a Test Entity" ; skos:inScheme . +ttm:test_case_type a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "test_case_type" ; + rdfs:range ttm:TestCaseTypeEnum ; + skos:definition "Type of TestCase." ; + skos:inScheme . + ttm:SemanticSeverityEnum a owl:Class, linkml:EnumDefinition ; owl:unionOf ( ) ; @@ -579,13 +599,12 @@ ttm:tags a owl:ObjectProperty, skos:definition "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; skos:inScheme . -ttm:EnvironmentEnum a owl:Class, - linkml:EnumDefinition ; - owl:unionOf ( ) ; - linkml:permissible_values , - , - , - . +ttm:test_assets a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "test_assets" ; + rdfs:range ttm:TestCase ; + skos:definition "List of explicitly enumerated Test Assets. The class attributes of TestAsset would be included in the TestCase versus being referred to by the identifier (curie) of the TestAsset. That is, this would be a list of objects (in JSONSchema serialization) versus a list of strings (where each string is an identifier pointing to another class)." ; + skos:inScheme . ttm:TestPersonaEnum a owl:Class, linkml:EnumDefinition ; @@ -595,14 +614,56 @@ ttm:TestPersonaEnum a owl:Class, , . -ttm:ExpectedOutputEnum a owl:Class, +ttm:TestCase a owl:Class, + linkml:ClassDefinition ; + rdfs:label "TestCase" ; + rdfs:subClassOf [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:tags ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:test_case_type ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:test_case_type ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:test_assets ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:tags ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:test_case_type ], + [ a owl:Restriction ; + owl:minCardinality 1 ; + owl:onProperty ttm:test_assets ], + ttm:TestEntity ; + skos:definition "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition." ; + skos:inScheme . + +ttm:TestCaseTypeEnum a owl:Class, linkml:EnumDefinition ; - owl:unionOf ( ) ; - linkml:permissible_values , - , - , - , - . + owl:unionOf ( ) ; + linkml:permissible_values , + , + , + , + . + +ttm:TestEntity a owl:Class, + linkml:ClassDefinition ; + rdfs:label "TestEntity" ; + skos:definition "Abstract global 'identification' class shared as a parent with all major model classes within the data model for Translator testing." ; + skos:inScheme . + +ttm:TestEnvEnum a owl:Class, + linkml:EnumDefinition ; + owl:unionOf ( ) ; + linkml:permissible_values , + , + , + . ttm:TestIssueEnum a owl:Class, linkml:EnumDefinition ; @@ -613,6 +674,16 @@ ttm:TestIssueEnum a owl:Class, , . +ttm:ExpectedOutputEnum a owl:Class, + linkml:EnumDefinition ; + owl:unionOf ( ) ; + linkml:permissible_values , + , + , + , + , + . + ttm:TestSourceEnum a owl:Class, linkml:EnumDefinition ; owl:unionOf ( ) ; @@ -624,12 +695,6 @@ ttm:TestSourceEnum a owl:Class, , . -ttm:TestEntity a owl:Class, - linkml:ClassDefinition ; - rdfs:label "TestEntity" ; - skos:definition "Abstract global 'identification' class shared as a parent with all major model classes within the data model for Translator testing." ; - skos:inScheme . - a owl:Ontology ; rdfs:label "Translator-Testing-Model" ; dcterms:license "MIT" ; diff --git a/project/protobuf/translator_testing_model.proto b/project/protobuf/translator_testing_model.proto index a920585..7efbe41 100644 --- a/project/protobuf/translator_testing_model.proto +++ b/project/protobuf/translator_testing_model.proto @@ -1,13 +1,43 @@ // Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. +message AcceptanceTestAsset + { + string name = 0 + string description = 0 + uriorcurie inputId = 0 + string inputName = 0 + string predicate = 0 + uriorcurie outputId = 0 + string outputName = 0 + expectedOutputEnum expectedOutput = 0 + testIssueEnum testIssue = 0 + semanticSeverityEnum semanticSeverity = 0 + boolean inV1 = 0 + boolean wellKnown = 0 + uriorcurie id = 0 + repeated string tags = 0 + date mustPassDate = 0 + testEnvEnum mustPassEnvironment = 0 + string query = 0 + string stringEntry = 0 + directionEnum direction = 0 + string answerInformalConcept = 0 + expectedResultsEnum expectedResult = 0 + integer topLevel = 0 + nodeEnum queryNode = 0 + string notes = 0 + } +// See AcceptanceTestAsset above for more details. message AcceptanceTestCase { uriorcurie id = 0 string name = 0 string description = 0 - repeated string tags = 0 + testEnvEnum testEnv = 0 + testCaseTypeEnum testCaseType = 0 + queryTypeEnum queryType = 0 repeated precondition preconditions = 0 - repeated semanticSmokeTestInput inputs = 0 - repeated semanticSmokeTestOutput outputs = 0 + repeated string tags = 0 + repeated acceptanceTestAsset testAssets = 0 } message AcceptanceTestSuite { @@ -20,84 +50,76 @@ message AcceptanceTestSuite repeated testCase testCases = 0 testCaseSpecification testCaseSpecification = 0 } -// Represents an input to a TestCase -message Input +// TRAPI and Biolink Model standards compliance test +message ComplianceTestCase { uriorcurie id = 0 string name = 0 string description = 0 + testEnvEnum testEnv = 0 + testCaseTypeEnum testCaseType = 0 + queryTypeEnum queryType = 0 + repeated testCase testAssets = 0 + repeated precondition preconditions = 0 repeated string tags = 0 } -// Test case for testing the integrity of "One Hop" knowledge graph retrievals sensa legacy SRI_Testing harness. -message OneHopTestSuite +// Knowledge Graph navigation integration test +message KnowledgeGraphNavigationTestCase { uriorcurie id = 0 string name = 0 string description = 0 + testEnvEnum testEnv = 0 + testCaseTypeEnum testCaseType = 0 + queryTypeEnum queryType = 0 + repeated testCase testAssets = 0 + repeated precondition preconditions = 0 repeated string tags = 0 - testMetadata testMetadata = 0 - testPersonaEnum testPersona = 0 - repeated testCase testCases = 0 - testCaseSpecification testCaseSpecification = 0 } -// Represents an output from a TestCase -message Output +// 'One Hop' Knowledge Graph navigation integration test +message OneHopTestCase { uriorcurie id = 0 string name = 0 string description = 0 + testEnvEnum testEnv = 0 + testCaseTypeEnum testCaseType = 0 + queryTypeEnum queryType = 0 + repeated testCase testAssets = 0 + repeated precondition preconditions = 0 repeated string tags = 0 } -// Represents a precondition for a TestCase -message Precondition +// Test case for testing the integrity of "One Hop" knowledge graph retrievals sensa legacy SRI_Testing harness. +message OneHopTestSuite { uriorcurie id = 0 string name = 0 string description = 0 repeated string tags = 0 + testMetadata testMetadata = 0 + testPersonaEnum testPersona = 0 + repeated testCase testCases = 0 + testCaseSpecification testCaseSpecification = 0 } -// Represents a QueryAnswerPair specification of a Test Asset -message QueryAnswerPair - { - string name = 0 - string description = 0 - uriorcurie inputId = 0 - string inputName = 0 - uriorcurie outputId = 0 - string outputName = 0 - expectedOutputEnum expectedOutput = 0 - testIssueEnum testIssue = 0 - semanticSeverityEnum semanticSeverity = 0 - boolean inV1 = 0 - boolean wellKnown = 0 - uriorcurie id = 0 - repeated string tags = 0 - } -// Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. -message SemanticSmokeTestInput +// Represents a precondition for a TestCase +message Precondition { uriorcurie id = 0 string name = 0 string description = 0 repeated string tags = 0 - date mustPassDate = 0 - environmentEnum mustPassEnvironment = 0 - string query = 0 - string stringEntry = 0 - directionEnum direction = 0 - string answerInformalConcept = 0 - expectedResultsEnum expectedResult = 0 - curie curie = 0 - string topLevel = 0 - string node = 0 - string notes = 0 } // Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. -message SemanticSmokeTestOutput +message QuantitativeTestCase { uriorcurie id = 0 string name = 0 string description = 0 + testEnvEnum testEnv = 0 + testCaseTypeEnum testCaseType = 0 + queryTypeEnum queryType = 0 + repeated testCase testAssets = 0 + repeated precondition preconditions = 0 repeated string tags = 0 } // Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model. @@ -119,6 +141,7 @@ message TestAsset string description = 0 uriorcurie inputId = 0 string inputName = 0 + string predicate = 0 uriorcurie outputId = 0 string outputName = 0 expectedOutputEnum expectedOutput = 0 @@ -129,25 +152,18 @@ message TestAsset uriorcurie id = 0 repeated string tags = 0 } -// Represents an ad hoc list of Test Assets. -message TestAssetCollection - { - uriorcurie id = 0 - string name = 0 - string description = 0 - repeated testCase testAssets = 0 - repeated string tags = 0 - } -// Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a particular test condition. +// Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition. message TestCase { uriorcurie id = 0 string name = 0 string description = 0 - repeated string tags = 0 - repeated input inputs = 0 - repeated output outputs = 0 + testEnvEnum testEnv = 0 + testCaseTypeEnum testCaseType = 0 + queryTypeEnum queryType = 0 + repeated testCase testAssets = 0 repeated precondition preconditions = 0 + repeated string tags = 0 } // Parameterized declaration of the Test Case generator which dynamically generates a collection of Test Cases from Test Assets, using applicable heuristics. message TestCaseSpecification @@ -157,13 +173,14 @@ message TestCaseSpecification string description = 0 repeated string tags = 0 } -// Represents a single Biolink Model compliant instance of an edge that can be used for testing. +// Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing. message TestEdgeData { string name = 0 string description = 0 uriorcurie inputId = 0 string inputName = 0 + string predicate = 0 uriorcurie outputId = 0 string outputName = 0 expectedOutputEnum expectedOutput = 0 diff --git a/project/shacl/translator_testing_model.shacl.ttl b/project/shacl/translator_testing_model.shacl.ttl index be85548..f6f7129 100644 --- a/project/shacl/translator_testing_model.shacl.ttl +++ b/project/shacl/translator_testing_model.shacl.ttl @@ -6,67 +6,58 @@ ttm:AcceptanceTestCase a sh:NodeShape ; sh:closed true ; - sh:description "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; + sh:description "See AcceptanceTestAsset above for more details." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:SemanticSmokeTestOutput ; + sh:property [ sh:class ttm:AcceptanceTestAsset ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; sh:minCount 1 ; sh:nodeKind sh:IRI ; - sh:order 1 ; - sh:path ttm:outputs ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:order 3 ; + sh:path ttm:test_assets ], + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 4 ; - sh:path schema1:name ], + sh:minCount 1 ; + sh:order 5 ; + sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 6 ; - sh:path schema1:additionalType ], + sh:description "A human-readable description for a Test Entity" ; + sh:maxCount 1 ; + sh:order 7 ; + sh:path schema1:description ], [ sh:class ttm:Precondition ; sh:nodeKind sh:IRI ; - sh:order 2 ; + sh:order 4 ; sh:path ttm:preconditions ], - [ sh:description "A unique identifier for a Test Entity" ; + [ sh:description "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" ; + sh:in ( "acceptance" "quantitative" "compliance" "kg_navigation" "one_hop" ) ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_case_type ], + [ sh:description "Deployment environment within which the associated TestSuite is run." ; + sh:in ( "dev" "ci" "test" "prod" ) ; sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 3 ; - sh:path schema1:identifier ], - [ sh:class ttm:SemanticSmokeTestInput ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; sh:order 0 ; - sh:path ttm:inputs ], + sh:path ttm:test_env ], + [ sh:description "Type of TestCase query." ; + sh:in ( "treats" ) ; + sh:maxCount 1 ; + sh:order 2 ; + sh:path ttm:query_type ], [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; + sh:order 8 ; + sh:path schema1:additionalType ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 5 ; - sh:path schema1:description ] ; + sh:order 6 ; + sh:path schema1:name ] ; sh:targetClass ttm:AcceptanceTestCase . ttm:AcceptanceTestSuite a sh:NodeShape ; sh:closed true ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:TestMetadata ; - sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; - sh:maxCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 0 ; - sh:path ttm:test_metadata ], - [ sh:class ttm:TestCase ; - sh:description "List of explicitly enumerated Test Cases." ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_cases ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 4 ; - sh:path schema1:identifier ], - [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 7 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; + sh:property [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; sh:order 5 ; @@ -86,20 +77,12 @@ ttm:AcceptanceTestSuite a sh:NodeShape ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; sh:order 6 ; - sh:path schema1:description ] ; - sh:targetClass ttm:AcceptanceTestSuite . - -ttm:BenchmarkTestSuite a sh:NodeShape ; - sh:closed true ; - sh:description "JsonObj(is_a='TestSuite')" ; - sh:ignoredProperties ( rdf:type ) ; - sh:targetClass ttm:BenchmarkTestSuite . - -ttm:OneHopTestSuite a sh:NodeShape ; - sh:closed true ; - sh:description "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; + sh:path schema1:description ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 7 ; + sh:path schema1:additionalType ], + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; sh:order 4 ; @@ -114,106 +97,175 @@ ttm:OneHopTestSuite a sh:NodeShape ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; sh:order 0 ; - sh:path ttm:test_metadata ], - [ sh:description "A Test persona describes the user or operational context of a given test." ; - sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; + sh:path ttm:test_metadata ] ; + sh:targetClass ttm:AcceptanceTestSuite . + +ttm:BenchmarkTestSuite a sh:NodeShape ; + sh:closed true ; + sh:description "JsonObj(is_a='TestSuite')" ; + sh:ignoredProperties ( rdf:type ) ; + sh:targetClass ttm:BenchmarkTestSuite . + +ttm:ComplianceTestCase a sh:NodeShape ; + sh:closed true ; + sh:description "TRAPI and Biolink Model standards compliance test" ; + sh:ignoredProperties ( rdf:type ) ; + sh:property [ sh:datatype xsd:string ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; + sh:order 8 ; + sh:path schema1:additionalType ], + [ sh:description "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" ; + sh:in ( "acceptance" "quantitative" "compliance" "kg_navigation" "one_hop" ) ; sh:maxCount 1 ; sh:order 1 ; - sh:path ttm:test_persona ], + sh:path ttm:test_case_type ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 5 ; + sh:order 6 ; sh:path schema1:name ], - [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 7 ; - sh:path schema1:additionalType ], + [ sh:description "Deployment environment within which the associated TestSuite is run." ; + sh:in ( "dev" "ci" "test" "prod" ) ; + sh:maxCount 1 ; + sh:order 0 ; + sh:path ttm:test_env ], + [ sh:class ttm:Precondition ; + sh:nodeKind sh:IRI ; + sh:order 4 ; + sh:path ttm:preconditions ], + [ sh:description "Type of TestCase query." ; + sh:in ( "treats" ) ; + sh:maxCount 1 ; + sh:order 2 ; + sh:path ttm:query_type ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 6 ; + sh:order 7 ; sh:path schema1:description ], - [ sh:class ttm:TestCaseSpecification ; - sh:description "Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)" ; - sh:maxCount 1 ; + [ sh:class ttm:TestCase ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; + sh:minCount 1 ; sh:nodeKind sh:IRI ; sh:order 3 ; - sh:path ttm:test_case_specification ] ; - sh:targetClass ttm:OneHopTestSuite . + sh:path ttm:test_assets ], + [ sh:description "A unique identifier for a Test Entity" ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:order 5 ; + sh:path schema1:identifier ] ; + sh:targetClass ttm:ComplianceTestCase . -ttm:QueryAnswerPair a sh:NodeShape ; +ttm:KnowledgeGraphNavigationTestCase a sh:NodeShape ; sh:closed true ; - sh:description "Represents a QueryAnswerPair specification of a Test Asset" ; + sh:description "Knowledge Graph navigation integration test" ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:boolean ; + sh:property [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 6 ; + sh:path schema1:name ], + [ sh:class ttm:TestCase ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 3 ; + sh:path ttm:test_assets ], + [ sh:description "Type of TestCase query." ; + sh:in ( "treats" ) ; sh:maxCount 1 ; + sh:order 2 ; + sh:path ttm:query_type ], + [ sh:description "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" ; + sh:in ( "acceptance" "quantitative" "compliance" "kg_navigation" "one_hop" ) ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_case_type ], + [ sh:datatype xsd:string ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; sh:order 8 ; - sh:path ttm:well_known ], + sh:path schema1:additionalType ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 9 ; - sh:path schema1:identifier ], - [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" ) ; - sh:maxCount 1 ; sh:order 5 ; - sh:path ttm:test_issue ], + sh:path schema1:identifier ], + [ sh:class ttm:Precondition ; + sh:nodeKind sh:IRI ; + sh:order 4 ; + sh:path ttm:preconditions ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 11 ; - sh:path schema1:description ], - [ sh:datatype xsd:boolean ; sh:maxCount 1 ; sh:order 7 ; - sh:path ttm:in_v1 ], - [ sh:datatype xsd:string ; + sh:path schema1:description ], + [ sh:description "Deployment environment within which the associated TestSuite is run." ; + sh:in ( "dev" "ci" "test" "prod" ) ; sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:input_name ], - [ sh:maxCount 1 ; - sh:order 2 ; - sh:path ttm:output_id ], - [ sh:maxCount 1 ; sh:order 0 ; - sh:path ttm:input_id ], - [ sh:in ( "High" "Low" "NotApplicable" ) ; + sh:path ttm:test_env ] ; + sh:targetClass ttm:KnowledgeGraphNavigationTestCase . + +ttm:OneHopTestCase a sh:NodeShape ; + sh:closed true ; + sh:description "'One Hop' Knowledge Graph navigation integration test" ; + sh:ignoredProperties ( rdf:type ) ; + sh:property [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:semantic_severity ], + sh:minCount 1 ; + sh:order 5 ; + sh:path schema1:identifier ], + [ sh:class ttm:Precondition ; + sh:nodeKind sh:IRI ; + sh:order 4 ; + sh:path ttm:preconditions ], [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection " ; - sh:order 12 ; - sh:path schema1:additionalType ], + sh:description "A human-readable description for a Test Entity" ; + sh:maxCount 1 ; + sh:order 7 ; + sh:path schema1:description ], + [ sh:description "Deployment environment within which the associated TestSuite is run." ; + sh:in ( "dev" "ci" "test" "prod" ) ; + sh:maxCount 1 ; + sh:order 0 ; + sh:path ttm:test_env ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 10 ; + sh:order 6 ; sh:path schema1:name ], - [ sh:in ( "Acceptable" "BadButForgivable" "NeverShow" "1_TopAnswer" "4_NeverShow" ) ; + [ sh:description "Type of TestCase query." ; + sh:in ( "treats" ) ; sh:maxCount 1 ; - sh:order 4 ; - sh:path ttm:expected_output ], + sh:order 2 ; + sh:path ttm:query_type ], [ sh:datatype xsd:string ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; + sh:order 8 ; + sh:path schema1:additionalType ], + [ sh:description "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" ; + sh:in ( "acceptance" "quantitative" "compliance" "kg_navigation" "one_hop" ) ; sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_case_type ], + [ sh:class ttm:TestCase ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; sh:order 3 ; - sh:path ttm:output_name ] ; - sh:targetClass ttm:QueryAnswerPair . + sh:path ttm:test_assets ] ; + sh:targetClass ttm:OneHopTestCase . -ttm:StandardsComplianceTestSuite a sh:NodeShape ; +ttm:OneHopTestSuite a sh:NodeShape ; sh:closed true ; - sh:description "Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model." ; + sh:description "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:TestCase ; - sh:description "List of explicitly enumerated Test Cases." ; + sh:property [ sh:class ttm:TestCaseSpecification ; + sh:description "Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)" ; + sh:maxCount 1 ; sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_cases ], - [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 7 ; - sh:path schema1:additionalType ], + sh:order 3 ; + sh:path ttm:test_case_specification ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; @@ -225,17 +277,11 @@ ttm:StandardsComplianceTestSuite a sh:NodeShape ; sh:nodeKind sh:IRI ; sh:order 0 ; sh:path ttm:test_metadata ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 6 ; - sh:path schema1:description ], - [ sh:class ttm:TestCaseSpecification ; - sh:description "Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)" ; - sh:maxCount 1 ; + [ sh:class ttm:TestCase ; + sh:description "List of explicitly enumerated Test Cases." ; sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:test_case_specification ], + sh:order 2 ; + sh:path ttm:test_cases ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; @@ -245,442 +291,459 @@ ttm:StandardsComplianceTestSuite a sh:NodeShape ; sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; sh:maxCount 1 ; sh:order 1 ; - sh:path ttm:test_persona ] ; - sh:targetClass ttm:StandardsComplianceTestSuite . + sh:path ttm:test_persona ], + [ sh:datatype xsd:string ; + sh:description "A human-readable description for a Test Entity" ; + sh:maxCount 1 ; + sh:order 6 ; + sh:path schema1:description ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 7 ; + sh:path schema1:additionalType ] ; + sh:targetClass ttm:OneHopTestSuite . -ttm:TestAsset a sh:NodeShape ; +ttm:QuantitativeTestCase a sh:NodeShape ; sh:closed true ; - sh:description "Represents a Test Asset, which is a single specific instance of TestCase-agnostic semantic parameters representing the specification of a Translator test target with inputs and (expected) outputs." ; + sh:description "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 3 ; - sh:path ttm:output_name ], - [ sh:maxCount 1 ; - sh:order 2 ; - sh:path ttm:output_id ], - [ sh:in ( "Acceptable" "BadButForgivable" "NeverShow" "1_TopAnswer" "4_NeverShow" ) ; - sh:maxCount 1 ; - sh:order 4 ; - sh:path ttm:expected_output ], + sh:order 7 ; + sh:path schema1:description ], [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:input_name ], - [ sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:input_id ], - [ sh:in ( "High" "Low" "NotApplicable" ) ; + sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; sh:order 6 ; - sh:path ttm:semantic_severity ], - [ sh:datatype xsd:boolean ; + sh:path schema1:name ], + [ sh:class ttm:Precondition ; + sh:nodeKind sh:IRI ; + sh:order 4 ; + sh:path ttm:preconditions ], + [ sh:description "Deployment environment within which the associated TestSuite is run." ; + sh:in ( "dev" "ci" "test" "prod" ) ; sh:maxCount 1 ; + sh:order 0 ; + sh:path ttm:test_env ], + [ sh:datatype xsd:string ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; sh:order 8 ; - sh:path ttm:well_known ], + sh:path schema1:additionalType ], + [ sh:description "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" ; + sh:in ( "acceptance" "quantitative" "compliance" "kg_navigation" "one_hop" ) ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_case_type ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 9 ; + sh:order 5 ; sh:path schema1:identifier ], - [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection " ; - sh:order 12 ; - sh:path schema1:additionalType ], - [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" ) ; + [ sh:description "Type of TestCase query." ; + sh:in ( "treats" ) ; sh:maxCount 1 ; - sh:order 5 ; - sh:path ttm:test_issue ], - [ sh:datatype xsd:string ; + sh:order 2 ; + sh:path ttm:query_type ], + [ sh:class ttm:TestCase ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 3 ; + sh:path ttm:test_assets ] ; + sh:targetClass ttm:QuantitativeTestCase . + +ttm:StandardsComplianceTestSuite a sh:NodeShape ; + sh:closed true ; + sh:description "Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model." ; + sh:ignoredProperties ( rdf:type ) ; + sh:property [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 11 ; + sh:order 6 ; sh:path schema1:description ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 7 ; + sh:path schema1:additionalType ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 10 ; + sh:order 5 ; sh:path schema1:name ], - [ sh:datatype xsd:boolean ; + [ sh:description "A Test persona describes the user or operational context of a given test." ; + sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:in_v1 ] ; - sh:targetClass ttm:TestAsset . - -ttm:TestAssetCollection a sh:NodeShape ; - sh:closed true ; - sh:description "Represents an ad hoc list of Test Assets." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; + sh:order 1 ; + sh:path ttm:test_persona ], + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 2 ; - sh:path schema1:identifier ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; sh:order 4 ; - sh:path schema1:description ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:path schema1:identifier ], + [ sh:class ttm:TestCase ; + sh:description "List of explicitly enumerated Test Cases." ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:test_cases ], + [ sh:class ttm:TestCaseSpecification ; + sh:description "Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)" ; sh:maxCount 1 ; + sh:nodeKind sh:IRI ; sh:order 3 ; - sh:path schema1:name ], - [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection " ; - sh:order 1 ; - sh:path schema1:additionalType ], - [ sh:class ttm:TestCase ; - sh:description "List of explicitly enumerated Test Assets." ; - sh:minCount 1 ; + sh:path ttm:test_case_specification ], + [ sh:class ttm:TestMetadata ; + sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; + sh:maxCount 1 ; sh:nodeKind sh:IRI ; sh:order 0 ; - sh:path ttm:test_assets ] ; - sh:targetClass ttm:TestAssetCollection . + sh:path ttm:test_metadata ] ; + sh:targetClass ttm:StandardsComplianceTestSuite . -ttm:TestEdgeData a sh:NodeShape ; +ttm:TestAsset a sh:NodeShape ; sh:closed true ; - sh:description "Represents a single Biolink Model compliant instance of an edge that can be used for testing." ; + sh:description "Represents a Test Asset, which is a single specific instance of TestCase-agnostic semantic parameters representing the specification of a Translator test target with inputs and (expected) outputs." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:boolean ; + sh:property [ sh:in ( "Top_Answer" "Acceptable" "BadButForgivable" "NeverShow" "1_TopAnswer" "4_NeverShow" ) ; sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:in_v1 ], - [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection " ; - sh:order 12 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 11 ; - sh:path schema1:description ], + sh:order 5 ; + sh:path ttm:expected_output ], [ sh:datatype xsd:string ; sh:maxCount 1 ; sh:order 1 ; sh:path ttm:input_name ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 3 ; - sh:path ttm:output_name ], [ sh:maxCount 1 ; - sh:order 2 ; - sh:path ttm:output_id ], - [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" ) ; + sh:order 0 ; + sh:path ttm:input_id ], + [ sh:datatype xsd:boolean ; sh:maxCount 1 ; - sh:order 5 ; - sh:path ttm:test_issue ], + sh:order 9 ; + sh:path ttm:well_known ], + [ sh:description "A unique identifier for a Test Entity" ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:order 10 ; + sh:path schema1:identifier ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 10 ; + sh:order 11 ; sh:path schema1:name ], - [ sh:in ( "High" "Low" "NotApplicable" ) ; + [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" ) ; sh:maxCount 1 ; sh:order 6 ; - sh:path ttm:semantic_severity ], - [ sh:in ( "Acceptable" "BadButForgivable" "NeverShow" "1_TopAnswer" "4_NeverShow" ) ; + sh:path ttm:test_issue ], + [ sh:datatype xsd:string ; sh:maxCount 1 ; sh:order 4 ; - sh:path ttm:expected_output ], - [ sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:input_id ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 9 ; - sh:path schema1:identifier ], + sh:path ttm:output_name ], [ sh:datatype xsd:boolean ; sh:maxCount 1 ; sh:order 8 ; - sh:path ttm:well_known ] ; - sh:targetClass ttm:TestEdgeData . - -ttm:TestEntity a sh:NodeShape ; - sh:closed false ; - sh:description "Abstract global 'identification' class shared as a parent with all major model classes within the data model for Translator testing." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:order 0 ; - sh:path schema1:identifier ], + sh:path ttm:in_v1 ], [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; sh:order 2 ; - sh:path schema1:description ], + sh:path ttm:predicate ], [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 3 ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection " ; + sh:order 13 ; sh:path schema1:additionalType ], + [ sh:maxCount 1 ; + sh:order 3 ; + sh:path ttm:output_id ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 1 ; - sh:path schema1:name ] ; - sh:targetClass ttm:TestEntity . + sh:order 12 ; + sh:path schema1:description ], + [ sh:in ( "High" "Low" "NotApplicable" ) ; + sh:maxCount 1 ; + sh:order 7 ; + sh:path ttm:semantic_severity ] ; + sh:targetClass ttm:TestAsset . -ttm:TestSuite a sh:NodeShape ; +ttm:TestEdgeData a sh:NodeShape ; sh:closed true ; - sh:description "Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present." ; + sh:description "Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing." ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 6 ; - sh:path schema1:description ], + sh:order 4 ; + sh:path ttm:output_name ], + [ sh:description "A unique identifier for a Test Entity" ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:order 10 ; + sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection " ; + sh:order 13 ; + sh:path schema1:additionalType ], + [ sh:in ( "Top_Answer" "Acceptable" "BadButForgivable" "NeverShow" "1_TopAnswer" "4_NeverShow" ) ; sh:maxCount 1 ; sh:order 5 ; - sh:path schema1:name ], + sh:path ttm:expected_output ], + [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" ) ; + sh:maxCount 1 ; + sh:order 6 ; + sh:path ttm:test_issue ], + [ sh:datatype xsd:boolean ; + sh:maxCount 1 ; + sh:order 8 ; + sh:path ttm:in_v1 ], [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 7 ; - sh:path schema1:additionalType ], - [ sh:description "A Test persona describes the user or operational context of a given test." ; - sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; sh:maxCount 1 ; sh:order 1 ; - sh:path ttm:test_persona ], - [ sh:class ttm:TestMetadata ; - sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; + sh:path ttm:input_name ], + [ sh:in ( "High" "Low" "NotApplicable" ) ; sh:maxCount 1 ; - sh:nodeKind sh:IRI ; + sh:order 7 ; + sh:path ttm:semantic_severity ], + [ sh:maxCount 1 ; sh:order 0 ; - sh:path ttm:test_metadata ], - [ sh:class ttm:TestCaseSpecification ; - sh:description "Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)" ; + sh:path ttm:input_id ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:test_case_specification ], - [ sh:description "A unique identifier for a Test Entity" ; + sh:order 11 ; + sh:path schema1:name ], + [ sh:datatype xsd:boolean ; + sh:maxCount 1 ; + sh:order 9 ; + sh:path ttm:well_known ], + [ sh:datatype xsd:string ; + sh:description "A human-readable description for a Test Entity" ; + sh:maxCount 1 ; + sh:order 12 ; + sh:path schema1:description ], + [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 4 ; - sh:path schema1:identifier ], - [ sh:class ttm:TestCase ; - sh:description "List of explicitly enumerated Test Cases." ; - sh:nodeKind sh:IRI ; sh:order 2 ; - sh:path ttm:test_cases ] ; - sh:targetClass ttm:TestSuite . + sh:path ttm:predicate ], + [ sh:maxCount 1 ; + sh:order 3 ; + sh:path ttm:output_id ] ; + sh:targetClass ttm:TestEdgeData . -ttm:Input a sh:NodeShape ; - sh:closed true ; - sh:description "Represents an input to a TestCase" ; +ttm:TestEntity a sh:NodeShape ; + sh:closed false ; + sh:description "Abstract global 'identification' class shared as a parent with all major model classes within the data model for Translator testing." ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 1 ; - sh:path schema1:name ], + sh:order 2 ; + sh:path schema1:description ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:minCount 1 ; sh:order 0 ; sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; + sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 2 ; - sh:path schema1:description ], + sh:order 1 ; + sh:path schema1:name ], [ sh:datatype xsd:string ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; sh:order 3 ; sh:path schema1:additionalType ] ; - sh:targetClass ttm:Input . + sh:targetClass ttm:TestEntity . -ttm:Output a sh:NodeShape ; - sh:closed true ; - sh:description "Represents an output from a TestCase" ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 3 ; - sh:path schema1:additionalType ], - [ sh:description "A unique identifier for a Test Entity" ; +ttm:TestSuite a sh:NodeShape ; + sh:closed true ; + sh:description "Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present." ; + sh:ignoredProperties ( rdf:type ) ; + sh:property [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 0 ; + sh:order 4 ; sh:path schema1:identifier ], + [ sh:description "A Test persona describes the user or operational context of a given test." ; + sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_persona ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 2 ; + sh:order 6 ; sh:path schema1:description ], + [ sh:class ttm:TestCaseSpecification ; + sh:description "Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)" ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 3 ; + sh:path ttm:test_case_specification ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 7 ; + sh:path schema1:additionalType ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 1 ; - sh:path schema1:name ] ; - sh:targetClass ttm:Output . + sh:order 5 ; + sh:path schema1:name ], + [ sh:class ttm:TestCase ; + sh:description "List of explicitly enumerated Test Cases." ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:test_cases ], + [ sh:class ttm:TestMetadata ; + sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 0 ; + sh:path ttm:test_metadata ] ; + sh:targetClass ttm:TestSuite . -ttm:SemanticSmokeTestInput a sh:NodeShape ; +ttm:AcceptanceTestAsset a sh:NodeShape ; sh:closed true ; sh:description "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; + sh:maxCount 1 ; + sh:order 11 ; + sh:path ttm:input_name ], + [ sh:datatype xsd:string ; sh:description "An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0" ; sh:maxCount 1 ; sh:order 5 ; sh:path ttm:answer_informal_concept ], - [ sh:description "The expected result of the query" ; - sh:in ( "include_good" "exclude_bad" ) ; - sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:expected_result ], - [ sh:datatype xsd:date ; - sh:description "The date by which this test must pass" ; + [ sh:in ( "High" "Low" "NotApplicable" ) ; sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:must_pass_date ], - [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 14 ; - sh:path schema1:additionalType ], + sh:order 17 ; + sh:path ttm:semantic_severity ], [ sh:datatype xsd:string ; - sh:description "The question a SME would ask" ; - sh:maxCount 1 ; - sh:order 2 ; - sh:path ttm:query ], - [ sh:description "A unique identifier for a Test Entity" ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 11 ; - sh:path schema1:identifier ], - [ sh:description "The environment in which this test must pass" ; - sh:in ( "DEV" "CI" "TEST" "PROD" ) ; + sh:order 22 ; + sh:path schema1:description ], + [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" ) ; sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:must_pass_environment ], + sh:order 16 ; + sh:path ttm:test_issue ], + [ sh:maxCount 1 ; + sh:order 10 ; + sh:path ttm:input_id ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; sh:order 12 ; - sh:path schema1:name ], - [ sh:datatype xsd:string ; - sh:description "The object of the core triple to be tested" ; - sh:maxCount 1 ; - sh:order 3 ; - sh:path ttm:string_entry ], - [ sh:description "The curie of the query" ; + sh:path ttm:predicate ], + [ sh:datatype xsd:date ; + sh:description "The date by which this test must pass" ; sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:curie ], + sh:order 0 ; + sh:path ttm:must_pass_date ], [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 13 ; - sh:path schema1:description ], + sh:order 14 ; + sh:path ttm:output_name ], + [ sh:description "The expected result of the query" ; + sh:in ( "include_good" "exclude_bad" ) ; + sh:maxCount 1 ; + sh:order 6 ; + sh:path ttm:expected_result ], [ sh:description "The direction of the expected query result triple" ; sh:in ( "increased" "decreased" ) ; sh:maxCount 1 ; sh:order 4 ; sh:path ttm:direction ], - [ sh:datatype xsd:string ; - sh:description "The node of the TRAPI query to replace." ; + [ sh:in ( "Top_Answer" "Acceptable" "BadButForgivable" "NeverShow" "1_TopAnswer" "4_NeverShow" ) ; sh:maxCount 1 ; - sh:order 9 ; - sh:path ttm:node ], - [ sh:datatype xsd:string ; + sh:order 15 ; + sh:path ttm:expected_output ], + [ sh:datatype xsd:integer ; sh:description "The answer must return in these many results" ; sh:maxCount 1 ; - sh:order 8 ; + sh:order 7 ; sh:path ttm:top_level ], [ sh:datatype xsd:string ; sh:description "The notes of the query" ; sh:maxCount 1 ; - sh:order 10 ; - sh:path ttm:notes ] ; - sh:targetClass ttm:SemanticSmokeTestInput . - -ttm:SemanticSmokeTestOutput a sh:NodeShape ; - sh:closed true ; - sh:description "Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:order 9 ; + sh:path ttm:notes ], + [ sh:datatype xsd:string ; + sh:description "The question a SME would ask" ; sh:maxCount 1 ; - sh:order 1 ; - sh:path schema1:name ], + sh:order 2 ; + sh:path ttm:query ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 0 ; + sh:order 20 ; sh:path schema1:identifier ], + [ sh:description "The node of the (templated) TRAPI query to replace" ; + sh:in ( "subject" "object" ) ; + sh:maxCount 1 ; + sh:order 8 ; + sh:path ttm:query_node ], + [ sh:maxCount 1 ; + sh:order 13 ; + sh:path ttm:output_id ], + [ sh:description "The deployment environment within which this test must pass." ; + sh:in ( "dev" "ci" "test" "prod" ) ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:must_pass_environment ], [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 3 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; + sh:description "The object of the core triple to be tested" ; sh:maxCount 1 ; - sh:order 2 ; - sh:path schema1:description ] ; - sh:targetClass ttm:SemanticSmokeTestOutput . - -ttm:Precondition a sh:NodeShape ; - sh:closed true ; - sh:description "Represents a precondition for a TestCase" ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; sh:order 3 ; - sh:path schema1:additionalType ], + sh:path ttm:string_entry ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 1 ; + sh:order 21 ; sh:path schema1:name ], - [ sh:description "A unique identifier for a Test Entity" ; + [ sh:datatype xsd:boolean ; sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 0 ; - sh:path schema1:identifier ], + sh:order 19 ; + sh:path ttm:well_known ], [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection " ; + sh:order 23 ; + sh:path schema1:additionalType ], + [ sh:datatype xsd:boolean ; sh:maxCount 1 ; - sh:order 2 ; - sh:path schema1:description ] ; - sh:targetClass ttm:Precondition . + sh:order 18 ; + sh:path ttm:in_v1 ] ; + sh:targetClass ttm:AcceptanceTestAsset . ttm:TestCaseSpecification a sh:NodeShape ; sh:closed true ; sh:description "Parameterized declaration of the Test Case generator which dynamically generates a collection of Test Cases from Test Assets, using applicable heuristics." ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path schema1:name ], + [ sh:datatype xsd:string ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; sh:order 3 ; sh:path schema1:additionalType ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 0 ; - sh:path schema1:identifier ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; sh:order 2 ; sh:path schema1:description ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 1 ; - sh:path schema1:name ] ; + sh:minCount 1 ; + sh:order 0 ; + sh:path schema1:identifier ] ; sh:targetClass ttm:TestCaseSpecification . ttm:TestMetadata a sh:NodeShape ; sh:closed true ; sh:description "Represents metadata related to (external SME, SMURF, Translator feedback, large scale batch, etc.) like the provenance of test assets, cases and/or suites." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; + sh:property [ sh:description "Provenance of a specific set of test assets, cases and/or suites." ; + sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 3 ; - sh:path schema1:identifier ], + sh:order 0 ; + sh:path ttm:test_source ], [ sh:description "Source documentation where original test particulars are registered (e.g. Github repo)" ; sh:maxCount 1 ; sh:order 1 ; @@ -689,21 +752,21 @@ ttm:TestMetadata a sh:NodeShape ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; sh:order 6 ; sh:path schema1:additionalType ], - [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; - sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 2 ; - sh:path ttm:test_objective ], + sh:minCount 1 ; + sh:order 3 ; + sh:path schema1:identifier ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; sh:order 4 ; sh:path schema1:name ], - [ sh:description "Provenance of a specific set of test assets, cases and/or suites." ; - sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; + [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; + sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:test_source ], + sh:order 2 ; + sh:path ttm:test_objective ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; @@ -711,40 +774,78 @@ ttm:TestMetadata a sh:NodeShape ; sh:path schema1:description ] ; sh:targetClass ttm:TestMetadata . -ttm:TestCase a sh:NodeShape ; +ttm:Precondition a sh:NodeShape ; sh:closed true ; - sh:description "Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a particular test condition." ; + sh:description "Represents a precondition for a TestCase" ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:Input ; - sh:nodeKind sh:IRI ; + sh:property [ sh:description "A unique identifier for a Test Entity" ; + sh:maxCount 1 ; + sh:minCount 1 ; sh:order 0 ; - sh:path ttm:inputs ], - [ sh:class ttm:Precondition ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:preconditions ], + sh:path schema1:identifier ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path schema1:name ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 3 ; + sh:path schema1:additionalType ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; + sh:order 2 ; + sh:path schema1:description ] ; + sh:targetClass ttm:Precondition . + +ttm:TestCase a sh:NodeShape ; + sh:closed true ; + sh:description "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition." ; + sh:ignoredProperties ( rdf:type ) ; + sh:property [ sh:datatype xsd:string ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; + sh:order 8 ; + sh:path schema1:additionalType ], + [ sh:description "A unique identifier for a Test Entity" ; + sh:maxCount 1 ; + sh:minCount 1 ; sh:order 5 ; - sh:path schema1:description ], - [ sh:class ttm:Output ; - sh:nodeKind sh:IRI ; + sh:path schema1:identifier ], + [ sh:description "Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)" ; + sh:in ( "acceptance" "quantitative" "compliance" "kg_navigation" "one_hop" ) ; + sh:maxCount 1 ; sh:order 1 ; - sh:path ttm:outputs ], + sh:path ttm:test_case_type ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; + sh:order 7 ; + sh:path schema1:description ], + [ sh:class ttm:Precondition ; + sh:nodeKind sh:IRI ; sh:order 4 ; - sh:path schema1:name ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; + sh:path ttm:preconditions ], + [ sh:class ttm:TestCase ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; sh:minCount 1 ; + sh:nodeKind sh:IRI ; sh:order 3 ; - sh:path schema1:identifier ], + sh:path ttm:test_assets ], + [ sh:description "Type of TestCase query." ; + sh:in ( "treats" ) ; + sh:maxCount 1 ; + sh:order 2 ; + sh:path ttm:query_type ], + [ sh:description "Deployment environment within which the associated TestSuite is run." ; + sh:in ( "dev" "ci" "test" "prod" ) ; + sh:maxCount 1 ; + sh:order 0 ; + sh:path ttm:test_env ], [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; sh:order 6 ; - sh:path schema1:additionalType ] ; + sh:path schema1:name ] ; sh:targetClass ttm:TestCase . diff --git a/project/shex/translator_testing_model.shex b/project/shex/translator_testing_model.shex index 331f34b..aff0dd0 100644 --- a/project/shex/translator_testing_model.shex +++ b/project/shex/translator_testing_model.shex @@ -43,11 +43,28 @@ linkml:Jsonpath xsd:string linkml:Sparqlpath xsd:string + CLOSED { + ( $ ( & ; + rdf:type [ ] ? ; + @linkml:Date ? ; + @ ? ; + @linkml:String ? ; + @linkml:String ? ; + @ ? ; + @linkml:String ? ; + @ ? ; + @linkml:Integer ? ; + @ ? ; + @linkml:String ? + ) ; + rdf:type [ ] + ) +} + CLOSED { ( $ ( & ; rdf:type [ ] ? ; - @ + ; - @ + + @ + ) ; rdf:type [ ] ) @@ -67,74 +84,53 @@ linkml:Sparqlpath xsd:string ) } - ( - CLOSED { - ( $ ( & ; - rdf:type [ ] ? - ) ; - rdf:type [ ] - ) - } OR @ -) - - CLOSED { - ( $ ( & ; - rdf:type [ ] ? + CLOSED { + ( $ ( & ; + rdf:type [ ] ? ) ; - rdf:type [ ] + rdf:type [ ] ) } - ( + ( CLOSED { - ( $ ( & ; - rdf:type [ ] ? + ( $ ( & ; + rdf:type [ ] ? ) ; - rdf:type [ ] + rdf:type [ ] ) - } OR @ + } OR @ ) - CLOSED { - ( $ ( & ; - rdf:type [ ] ? + CLOSED { + ( $ ( & ; + rdf:type [ ] ? ) ; - rdf:type [ ] + rdf:type [ ] ) } - CLOSED { - ( $ ( & ; - rdf:type [ ] ? + CLOSED { + ( $ ( & ; + rdf:type [ ] ? ) ; - rdf:type [ ] + rdf:type [ ] ) } - CLOSED { - ( $ ( & ; - rdf:type [ ] ? ; - @linkml:Date ? ; - @ ? ; - @linkml:String ? ; - @linkml:String ? ; - @ ? ; - @linkml:String ? ; - @ ? ; - @linkml:Curie ? ; - @linkml:String ? ; - @linkml:String ? ; - @linkml:String ? + CLOSED { + ( $ ( & ; + rdf:type [ ] ? ) ; - rdf:type [ ] + rdf:type [ ] ) } - CLOSED { - ( $ ( & ; - rdf:type [ ] ? + CLOSED { + ( $ ( & ; + rdf:type [ ] ? ) ; - rdf:type [ ] + rdf:type [ ] ) } @@ -152,6 +148,7 @@ linkml:Sparqlpath xsd:string rdf:type [ ] ? ; @linkml:Uriorcurie ? ; @linkml:String ? ; + @linkml:String ? ; @linkml:Uriorcurie ? ; @linkml:String ? ; @ ? ; @@ -163,30 +160,23 @@ linkml:Sparqlpath xsd:string ) ; rdf:type [ ] ) - } OR @ OR @ + } OR @ OR @ ) - CLOSED { - ( $ ( & ; - rdf:type [ ] ? ; - @ + ; - schema1:additionalType @linkml:String * - ) ; - rdf:type [ ] - ) -} - ( CLOSED { ( $ ( & ; rdf:type [ ] ? ; - @ * ; - @ * ; - @ * + @ ? ; + @ ? ; + @ ? ; + @ + ; + @ * ; + schema1:additionalType @linkml:String * ) ; rdf:type [ ] ) - } OR @ + } OR @ OR @ OR @ OR @ ) CLOSED { @@ -206,8 +196,7 @@ linkml:Sparqlpath xsd:string } ( - @ OR @ OR @ OR @ OR @ OR @ OR @ - OR @ OR @ + @ OR @ OR @ OR @ OR @ OR @ ) { diff --git a/project/sqlschema/translator_testing_model.sql b/project/sqlschema/translator_testing_model.sql index c3c75c7..463e358 100644 --- a/project/sqlschema/translator_testing_model.sql +++ b/project/sqlschema/translator_testing_model.sql @@ -4,30 +4,65 @@ CREATE TABLE "AcceptanceTestCase" ( id TEXT NOT NULL, name TEXT, description TEXT, + test_env VARCHAR(4), + test_case_type VARCHAR(13), + query_type VARCHAR(6), preconditions TEXT, PRIMARY KEY (id) ); -CREATE TABLE "Precondition" ( +CREATE TABLE "ComplianceTestCase" ( id TEXT NOT NULL, name TEXT, description TEXT, + test_env VARCHAR(4), + test_case_type VARCHAR(13), + query_type VARCHAR(6), + test_assets TEXT NOT NULL, + preconditions TEXT, PRIMARY KEY (id) ); -CREATE TABLE "QueryAnswerPair" ( +CREATE TABLE "KnowledgeGraphNavigationTestCase" ( + id TEXT NOT NULL, name TEXT, description TEXT, - input_id TEXT, - input_name TEXT, - output_id TEXT, - output_name TEXT, - expected_output VARCHAR(16), - test_issue VARCHAR(20), - semantic_severity VARCHAR(13), - in_v1 BOOLEAN, - well_known BOOLEAN, + test_env VARCHAR(4), + test_case_type VARCHAR(13), + query_type VARCHAR(6), + test_assets TEXT NOT NULL, + preconditions TEXT, + PRIMARY KEY (id) +); + +CREATE TABLE "OneHopTestCase" ( id TEXT NOT NULL, + name TEXT, + description TEXT, + test_env VARCHAR(4), + test_case_type VARCHAR(13), + query_type VARCHAR(6), + test_assets TEXT NOT NULL, + preconditions TEXT, + PRIMARY KEY (id) +); + +CREATE TABLE "Precondition" ( + id TEXT NOT NULL, + name TEXT, + description TEXT, + PRIMARY KEY (id) +); + +CREATE TABLE "QuantitativeTestCase" ( + id TEXT NOT NULL, + name TEXT, + description TEXT, + test_env VARCHAR(4), + test_case_type VARCHAR(13), + query_type VARCHAR(6), + test_assets TEXT NOT NULL, + preconditions TEXT, PRIMARY KEY (id) ); @@ -36,6 +71,7 @@ CREATE TABLE "TestAsset" ( description TEXT, input_id TEXT, input_name TEXT, + predicate TEXT, output_id TEXT, output_name TEXT, expected_output VARCHAR(16), @@ -47,18 +83,14 @@ CREATE TABLE "TestAsset" ( PRIMARY KEY (id) ); -CREATE TABLE "TestAssetCollection" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - test_assets TEXT NOT NULL, - PRIMARY KEY (id) -); - CREATE TABLE "TestCase" ( id TEXT NOT NULL, name TEXT, description TEXT, + test_env VARCHAR(4), + test_case_type VARCHAR(13), + query_type VARCHAR(6), + test_assets TEXT NOT NULL, preconditions TEXT, PRIMARY KEY (id) ); @@ -75,6 +107,7 @@ CREATE TABLE "TestEdgeData" ( description TEXT, input_id TEXT, input_name TEXT, + predicate TEXT, output_id TEXT, output_name TEXT, expected_output VARCHAR(16), @@ -96,6 +129,35 @@ CREATE TABLE "TestMetadata" ( PRIMARY KEY (id) ); +CREATE TABLE "AcceptanceTestAsset" ( + name TEXT, + description TEXT, + input_id TEXT, + input_name TEXT, + predicate TEXT, + output_id TEXT, + output_name TEXT, + expected_output VARCHAR(16), + test_issue VARCHAR(20), + semantic_severity VARCHAR(13), + in_v1 BOOLEAN, + well_known BOOLEAN, + id TEXT NOT NULL, + must_pass_date DATE, + must_pass_environment VARCHAR(4), + "query" TEXT, + string_entry TEXT, + direction VARCHAR(9), + answer_informal_concept TEXT, + expected_result VARCHAR(12), + top_level INTEGER, + query_node VARCHAR(7), + notes TEXT, + "AcceptanceTestCase_id" TEXT, + PRIMARY KEY (id), + FOREIGN KEY("AcceptanceTestCase_id") REFERENCES "AcceptanceTestCase" (id) +); + CREATE TABLE "AcceptanceTestSuite" ( id TEXT NOT NULL, name TEXT, @@ -109,15 +171,6 @@ CREATE TABLE "AcceptanceTestSuite" ( FOREIGN KEY(test_case_specification) REFERENCES "TestCaseSpecification" (id) ); -CREATE TABLE "Input" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - "TestCase_id" TEXT, - PRIMARY KEY (id), - FOREIGN KEY("TestCase_id") REFERENCES "TestCase" (id) -); - CREATE TABLE "OneHopTestSuite" ( id TEXT NOT NULL, name TEXT, @@ -131,44 +184,6 @@ CREATE TABLE "OneHopTestSuite" ( FOREIGN KEY(test_case_specification) REFERENCES "TestCaseSpecification" (id) ); -CREATE TABLE "Output" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - "TestCase_id" TEXT, - PRIMARY KEY (id), - FOREIGN KEY("TestCase_id") REFERENCES "TestCase" (id) -); - -CREATE TABLE "SemanticSmokeTestInput" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - must_pass_date DATE, - must_pass_environment VARCHAR(4), - "query" TEXT, - string_entry TEXT, - direction VARCHAR(9), - answer_informal_concept TEXT, - expected_result VARCHAR(12), - curie TEXT, - top_level TEXT, - node TEXT, - notes TEXT, - "AcceptanceTestCase_id" TEXT, - PRIMARY KEY (id), - FOREIGN KEY("AcceptanceTestCase_id") REFERENCES "AcceptanceTestCase" (id) -); - -CREATE TABLE "SemanticSmokeTestOutput" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - "AcceptanceTestCase_id" TEXT, - PRIMARY KEY (id), - FOREIGN KEY("AcceptanceTestCase_id") REFERENCES "AcceptanceTestCase" (id) -); - CREATE TABLE "StandardsComplianceTestSuite" ( id TEXT NOT NULL, name TEXT, @@ -202,102 +217,95 @@ CREATE TABLE "AcceptanceTestCase_tags" ( FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestCase" (id) ); -CREATE TABLE "Precondition_tags" ( +CREATE TABLE "ComplianceTestCase_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "Precondition" (id) + FOREIGN KEY(backref_id) REFERENCES "ComplianceTestCase" (id) ); -CREATE TABLE "QueryAnswerPair_tags" ( +CREATE TABLE "KnowledgeGraphNavigationTestCase_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "QueryAnswerPair" (id) + FOREIGN KEY(backref_id) REFERENCES "KnowledgeGraphNavigationTestCase" (id) ); -CREATE TABLE "TestAsset_tags" ( +CREATE TABLE "OneHopTestCase_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "TestAsset" (id) + FOREIGN KEY(backref_id) REFERENCES "OneHopTestCase" (id) ); -CREATE TABLE "TestAssetCollection_tags" ( - backref_id TEXT, - tags TEXT, - PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "TestAssetCollection" (id) -); - -CREATE TABLE "TestCase_tags" ( +CREATE TABLE "Precondition_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "TestCase" (id) + FOREIGN KEY(backref_id) REFERENCES "Precondition" (id) ); -CREATE TABLE "TestCaseSpecification_tags" ( +CREATE TABLE "QuantitativeTestCase_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "TestCaseSpecification" (id) + FOREIGN KEY(backref_id) REFERENCES "QuantitativeTestCase" (id) ); -CREATE TABLE "TestEdgeData_tags" ( +CREATE TABLE "TestAsset_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "TestEdgeData" (id) + FOREIGN KEY(backref_id) REFERENCES "TestAsset" (id) ); -CREATE TABLE "TestMetadata_tags" ( +CREATE TABLE "TestCase_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "TestMetadata" (id) + FOREIGN KEY(backref_id) REFERENCES "TestCase" (id) ); -CREATE TABLE "AcceptanceTestSuite_tags" ( +CREATE TABLE "TestCaseSpecification_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestSuite" (id) + FOREIGN KEY(backref_id) REFERENCES "TestCaseSpecification" (id) ); -CREATE TABLE "Input_tags" ( +CREATE TABLE "TestEdgeData_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "Input" (id) + FOREIGN KEY(backref_id) REFERENCES "TestEdgeData" (id) ); -CREATE TABLE "OneHopTestSuite_tags" ( +CREATE TABLE "TestMetadata_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "OneHopTestSuite" (id) + FOREIGN KEY(backref_id) REFERENCES "TestMetadata" (id) ); -CREATE TABLE "Output_tags" ( +CREATE TABLE "AcceptanceTestAsset_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "Output" (id) + FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestAsset" (id) ); -CREATE TABLE "SemanticSmokeTestInput_tags" ( +CREATE TABLE "AcceptanceTestSuite_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "SemanticSmokeTestInput" (id) + FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestSuite" (id) ); -CREATE TABLE "SemanticSmokeTestOutput_tags" ( +CREATE TABLE "OneHopTestSuite_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "SemanticSmokeTestOutput" (id) + FOREIGN KEY(backref_id) REFERENCES "OneHopTestSuite" (id) ); CREATE TABLE "StandardsComplianceTestSuite_tags" ( diff --git a/src/translator_testing_model/datamodel/pydanticmodel.py b/src/translator_testing_model/datamodel/pydanticmodel.py index 442f81b..a53fa3d 100644 --- a/src/translator_testing_model/datamodel/pydanticmodel.py +++ b/src/translator_testing_model/datamodel/pydanticmodel.py @@ -69,11 +69,22 @@ class TestPersonaEnum(str, Enum): +class QueryTypeEnum(str, Enum): + """ + Query + """ + + treats = "treats" + + + class ExpectedOutputEnum(str, Enum): """ Expected output values for instances of Test Asset or Test Cases(?). (Note: does this Enum overlap with 'ExpectedResultsEnum' below?) """ + Top_Answer = "Top_Answer" + Acceptable = "Acceptable" BadButForgivable = "BadButForgivable" @@ -97,11 +108,54 @@ class ExpectedResultsEnum(str, Enum): +class NodeEnum(str, Enum): + """ + Target node of a Subject-Predicate-Object driven query + """ + + subject = "subject" + + object = "object" + + + +class TestEnvEnum(str, Enum): + """ + Testing environments within which a TestSuite is run by a TestRunner scheduled by the TestHarness. + """ + # Development + dev = "dev" + # Continuous Integration + ci = "ci" + # Test + test = "test" + # Production + prod = "prod" + + + +class TestCaseTypeEnum(str, Enum): + """ + Enumerated tags for types of test (generally applied to a TestCase). + """ + # Acceptance test + acceptance = "acceptance" + # Quantitative test + quantitative = "quantitative" + # Standards compliance test + compliance = "compliance" + # Knowledge Graph navigation integration test + kg_navigation = "kg_navigation" + # One Hop navigation test + one_hop = "one_hop" + + + class TestIssueEnum(str, Enum): causes_not_treats = "causes not treats" - # Text Mining Knowledge Provider generated relationship? + # 'Text Mining Knowledge Provider' generated relationship? TMKP = "TMKP" category_too_generic = "category too generic" @@ -134,19 +188,6 @@ class DirectionEnum(str, Enum): -class EnvironmentEnum(str, Enum): - - - DEV = "DEV" - - CI = "CI" - - TEST = "TEST" - - PROD = "PROD" - - - class TestEntity(ConfiguredBaseModel): """ Abstract global 'identification' class shared as a parent with all major model classes within the data model for Translator testing. @@ -176,6 +217,7 @@ class TestAsset(TestEntity): """ input_id: Optional[str] = Field(None) input_name: Optional[str] = Field(None) + predicate: Optional[str] = Field(None) output_id: Optional[str] = Field(None) output_name: Optional[str] = Field(None) expected_output: Optional[ExpectedOutputEnum] = Field(None) @@ -189,23 +231,23 @@ class TestAsset(TestEntity): tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection """) -class TestAssetCollection(TestEntity): - """ - Represents an ad hoc list of Test Assets. - """ - test_assets: Dict[str, TestCase] = Field(default_factory=dict, description="""List of explicitly enumerated Test Assets.""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection """) - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - - -class QueryAnswerPair(TestAsset): +class AcceptanceTestAsset(TestAsset): """ - Represents a QueryAnswerPair specification of a Test Asset + Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. """ + must_pass_date: Optional[date] = Field(None, description="""The date by which this test must pass""") + must_pass_environment: Optional[TestEnvEnum] = Field(None, description="""The deployment environment within which this test must pass.""") + query: Optional[str] = Field(None, description="""The question a SME would ask""") + string_entry: Optional[str] = Field(None, description="""The object of the core triple to be tested""") + direction: Optional[DirectionEnum] = Field(None, description="""The direction of the expected query result triple""") + answer_informal_concept: Optional[str] = Field(None, description="""An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0""") + expected_result: Optional[ExpectedResultsEnum] = Field(None, description="""The expected result of the query""") + top_level: Optional[int] = Field(None, description="""The answer must return in these many results""") + query_node: Optional[NodeEnum] = Field(None, description="""The node of the (templated) TRAPI query to replace""") + notes: Optional[str] = Field(None, description="""The notes of the query""") input_id: Optional[str] = Field(None) input_name: Optional[str] = Field(None) + predicate: Optional[str] = Field(None) output_id: Optional[str] = Field(None) output_name: Optional[str] = Field(None) expected_output: Optional[ExpectedOutputEnum] = Field(None) @@ -221,10 +263,11 @@ class QueryAnswerPair(TestAsset): class TestEdgeData(TestAsset): """ - Represents a single Biolink Model compliant instance of an edge that can be used for testing. + Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing. """ input_id: Optional[str] = Field(None) input_name: Optional[str] = Field(None) + predicate: Optional[str] = Field(None) output_id: Optional[str] = Field(None) output_name: Optional[str] = Field(None) expected_output: Optional[ExpectedOutputEnum] = Field(None) @@ -240,15 +283,17 @@ class TestEdgeData(TestAsset): class TestCase(TestEntity): """ - Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a particular test condition. + Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition. """ - inputs: Optional[List[str]] = Field(default_factory=list) - outputs: Optional[List[str]] = Field(default_factory=list) + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") class TestCaseSpecification(TestEntity): @@ -261,69 +306,84 @@ class TestCaseSpecification(TestEntity): tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") -class TestSuite(TestEntity): +class AcceptanceTestCase(TestCase): """ - Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present. + See AcceptanceTestAsset above for more details. """ - test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") - test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") - test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") - test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[AcceptanceTestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") + preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class AcceptanceTestCase(TestCase): +class QuantitativeTestCase(TestCase): """ Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. """ - inputs: List[str] = Field(default_factory=list) - outputs: List[str] = Field(default_factory=list) + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class AcceptanceTestSuite(TestSuite): - - test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") - test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") - test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") - test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") +class ComplianceTestCase(TestCase): + """ + TRAPI and Biolink Model standards compliance test + """ + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") + preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class BenchmarkTestSuite(ConfiguredBaseModel): +class KnowledgeGraphNavigationTestCase(TestCase): """ - JsonObj(is_a='TestSuite') + Knowledge Graph navigation integration test """ - None + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") + preconditions: Optional[List[str]] = Field(default_factory=list) + id: str = Field(..., description="""A unique identifier for a Test Entity""") + name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") + description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class StandardsComplianceTestSuite(TestSuite): +class OneHopTestCase(KnowledgeGraphNavigationTestCase): """ - Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model. + 'One Hop' Knowledge Graph navigation integration test """ - test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") - test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") - test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") - test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") + preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class OneHopTestSuite(TestSuite): +class TestSuite(TestEntity): """ - Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness. + Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present. """ test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") @@ -335,51 +395,47 @@ class OneHopTestSuite(TestSuite): tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") -class Input(TestEntity): - """ - Represents an input to a TestCase - """ +class AcceptanceTestSuite(TestSuite): + + test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") + test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") + test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") + test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") -class SemanticSmokeTestInput(Input): +class BenchmarkTestSuite(ConfiguredBaseModel): """ - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + JsonObj(is_a='TestSuite') """ - must_pass_date: Optional[date] = Field(None, description="""The date by which this test must pass""") - must_pass_environment: Optional[EnvironmentEnum] = Field(None, description="""The environment in which this test must pass""") - query: Optional[str] = Field(None, description="""The question a SME would ask""") - string_entry: Optional[str] = Field(None, description="""The object of the core triple to be tested""") - direction: Optional[DirectionEnum] = Field(None, description="""The direction of the expected query result triple""") - answer_informal_concept: Optional[str] = Field(None, description="""An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0""") - expected_result: Optional[ExpectedResultsEnum] = Field(None, description="""The expected result of the query""") - curie: Optional[str] = Field(None, description="""The curie of the query""") - top_level: Optional[str] = Field(None, description="""The answer must return in these many results""") - node: Optional[str] = Field(None, description="""The node of the TRAPI query to replace.""") - notes: Optional[str] = Field(None, description="""The notes of the query""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + None -class Output(TestEntity): +class StandardsComplianceTestSuite(TestSuite): """ - Represents an output from a TestCase + Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model. """ + test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") + test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") + test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") + test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") -class SemanticSmokeTestOutput(Output): +class OneHopTestSuite(TestSuite): """ - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness. """ + test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") + test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") + test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") + test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") @@ -402,20 +458,19 @@ class Precondition(TestEntity): TestEntity.update_forward_refs() TestMetadata.update_forward_refs() TestAsset.update_forward_refs() -TestAssetCollection.update_forward_refs() -QueryAnswerPair.update_forward_refs() +AcceptanceTestAsset.update_forward_refs() TestEdgeData.update_forward_refs() TestCase.update_forward_refs() TestCaseSpecification.update_forward_refs() -TestSuite.update_forward_refs() AcceptanceTestCase.update_forward_refs() +QuantitativeTestCase.update_forward_refs() +ComplianceTestCase.update_forward_refs() +KnowledgeGraphNavigationTestCase.update_forward_refs() +OneHopTestCase.update_forward_refs() +TestSuite.update_forward_refs() AcceptanceTestSuite.update_forward_refs() BenchmarkTestSuite.update_forward_refs() StandardsComplianceTestSuite.update_forward_refs() OneHopTestSuite.update_forward_refs() -Input.update_forward_refs() -SemanticSmokeTestInput.update_forward_refs() -Output.update_forward_refs() -SemanticSmokeTestOutput.update_forward_refs() Precondition.update_forward_refs() diff --git a/src/translator_testing_model/datamodel/pydanticmodel_v2.py b/src/translator_testing_model/datamodel/pydanticmodel_v2.py index 735c62e..7c8572e 100644 --- a/src/translator_testing_model/datamodel/pydanticmodel_v2.py +++ b/src/translator_testing_model/datamodel/pydanticmodel_v2.py @@ -65,11 +65,22 @@ class TestPersonaEnum(str, Enum): +class QueryTypeEnum(str, Enum): + """ + Query + """ + + treats = "treats" + + + class ExpectedOutputEnum(str, Enum): """ Expected output values for instances of Test Asset or Test Cases(?). (Note: does this Enum overlap with 'ExpectedResultsEnum' below?) """ + Top_Answer = "Top_Answer" + Acceptable = "Acceptable" BadButForgivable = "BadButForgivable" @@ -93,11 +104,54 @@ class ExpectedResultsEnum(str, Enum): +class NodeEnum(str, Enum): + """ + Target node of a Subject-Predicate-Object driven query + """ + + subject = "subject" + + object = "object" + + + +class TestEnvEnum(str, Enum): + """ + Testing environments within which a TestSuite is run by a TestRunner scheduled by the TestHarness. + """ + # Development + dev = "dev" + # Continuous Integration + ci = "ci" + # Test + test = "test" + # Production + prod = "prod" + + + +class TestCaseTypeEnum(str, Enum): + """ + Enumerated tags for types of test (generally applied to a TestCase). + """ + # Acceptance test + acceptance = "acceptance" + # Quantitative test + quantitative = "quantitative" + # Standards compliance test + compliance = "compliance" + # Knowledge Graph navigation integration test + kg_navigation = "kg_navigation" + # One Hop navigation test + one_hop = "one_hop" + + + class TestIssueEnum(str, Enum): causes_not_treats = "causes not treats" - # Text Mining Knowledge Provider generated relationship? + # 'Text Mining Knowledge Provider' generated relationship? TMKP = "TMKP" category_too_generic = "category too generic" @@ -130,19 +184,6 @@ class DirectionEnum(str, Enum): -class EnvironmentEnum(str, Enum): - - - DEV = "DEV" - - CI = "CI" - - TEST = "TEST" - - PROD = "PROD" - - - class TestEntity(ConfiguredBaseModel): """ Abstract global 'identification' class shared as a parent with all major model classes within the data model for Translator testing. @@ -172,6 +213,7 @@ class TestAsset(TestEntity): """ input_id: Optional[str] = Field(None) input_name: Optional[str] = Field(None) + predicate: Optional[str] = Field(None) output_id: Optional[str] = Field(None) output_name: Optional[str] = Field(None) expected_output: Optional[ExpectedOutputEnum] = Field(None) @@ -185,23 +227,23 @@ class TestAsset(TestEntity): tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection """) -class TestAssetCollection(TestEntity): - """ - Represents an ad hoc list of Test Assets. - """ - test_assets: Dict[str, TestCase] = Field(default_factory=dict, description="""List of explicitly enumerated Test Assets.""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection """) - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - - -class QueryAnswerPair(TestAsset): +class AcceptanceTestAsset(TestAsset): """ - Represents a QueryAnswerPair specification of a Test Asset + Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. """ + must_pass_date: Optional[date] = Field(None, description="""The date by which this test must pass""") + must_pass_environment: Optional[TestEnvEnum] = Field(None, description="""The deployment environment within which this test must pass.""") + query: Optional[str] = Field(None, description="""The question a SME would ask""") + string_entry: Optional[str] = Field(None, description="""The object of the core triple to be tested""") + direction: Optional[DirectionEnum] = Field(None, description="""The direction of the expected query result triple""") + answer_informal_concept: Optional[str] = Field(None, description="""An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0""") + expected_result: Optional[ExpectedResultsEnum] = Field(None, description="""The expected result of the query""") + top_level: Optional[int] = Field(None, description="""The answer must return in these many results""") + query_node: Optional[NodeEnum] = Field(None, description="""The node of the (templated) TRAPI query to replace""") + notes: Optional[str] = Field(None, description="""The notes of the query""") input_id: Optional[str] = Field(None) input_name: Optional[str] = Field(None) + predicate: Optional[str] = Field(None) output_id: Optional[str] = Field(None) output_name: Optional[str] = Field(None) expected_output: Optional[ExpectedOutputEnum] = Field(None) @@ -217,10 +259,11 @@ class QueryAnswerPair(TestAsset): class TestEdgeData(TestAsset): """ - Represents a single Biolink Model compliant instance of an edge that can be used for testing. + Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing. """ input_id: Optional[str] = Field(None) input_name: Optional[str] = Field(None) + predicate: Optional[str] = Field(None) output_id: Optional[str] = Field(None) output_name: Optional[str] = Field(None) expected_output: Optional[ExpectedOutputEnum] = Field(None) @@ -236,15 +279,17 @@ class TestEdgeData(TestAsset): class TestCase(TestEntity): """ - Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a particular test condition. + Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition. """ - inputs: Optional[List[str]] = Field(default_factory=list) - outputs: Optional[List[str]] = Field(default_factory=list) + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") class TestCaseSpecification(TestEntity): @@ -257,69 +302,84 @@ class TestCaseSpecification(TestEntity): tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") -class TestSuite(TestEntity): +class AcceptanceTestCase(TestCase): """ - Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present. + See AcceptanceTestAsset above for more details. """ - test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") - test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") - test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") - test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[AcceptanceTestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") + preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class AcceptanceTestCase(TestCase): +class QuantitativeTestCase(TestCase): """ Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. """ - inputs: List[str] = Field(default_factory=list) - outputs: List[str] = Field(default_factory=list) + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class AcceptanceTestSuite(TestSuite): - - test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") - test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") - test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") - test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") +class ComplianceTestCase(TestCase): + """ + TRAPI and Biolink Model standards compliance test + """ + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") + preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class BenchmarkTestSuite(ConfiguredBaseModel): +class KnowledgeGraphNavigationTestCase(TestCase): """ - JsonObj(is_a='TestSuite') + Knowledge Graph navigation integration test """ - None + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") + preconditions: Optional[List[str]] = Field(default_factory=list) + id: str = Field(..., description="""A unique identifier for a Test Entity""") + name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") + description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class StandardsComplianceTestSuite(TestSuite): +class OneHopTestCase(KnowledgeGraphNavigationTestCase): """ - Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model. + 'One Hop' Knowledge Graph navigation integration test """ - test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") - test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") - test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") - test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") + test_case_type: Optional[TestCaseTypeEnum] = Field(None, description="""Is this valid or even necessary with the class names already available from classes of 'test_assets' used (seems redundant?)""") + query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") + test_assets: List[TestCase] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") + preconditions: Optional[List[str]] = Field(default_factory=list) id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") -class OneHopTestSuite(TestSuite): +class TestSuite(TestEntity): """ - Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness. + Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present. """ test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") @@ -331,51 +391,47 @@ class OneHopTestSuite(TestSuite): tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") -class Input(TestEntity): - """ - Represents an input to a TestCase - """ +class AcceptanceTestSuite(TestSuite): + + test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") + test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") + test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") + test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") -class SemanticSmokeTestInput(Input): +class BenchmarkTestSuite(ConfiguredBaseModel): """ - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + JsonObj(is_a='TestSuite') """ - must_pass_date: Optional[date] = Field(None, description="""The date by which this test must pass""") - must_pass_environment: Optional[EnvironmentEnum] = Field(None, description="""The environment in which this test must pass""") - query: Optional[str] = Field(None, description="""The question a SME would ask""") - string_entry: Optional[str] = Field(None, description="""The object of the core triple to be tested""") - direction: Optional[DirectionEnum] = Field(None, description="""The direction of the expected query result triple""") - answer_informal_concept: Optional[str] = Field(None, description="""An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0""") - expected_result: Optional[ExpectedResultsEnum] = Field(None, description="""The expected result of the query""") - curie: Optional[str] = Field(None, description="""The curie of the query""") - top_level: Optional[str] = Field(None, description="""The answer must return in these many results""") - node: Optional[str] = Field(None, description="""The node of the TRAPI query to replace.""") - notes: Optional[str] = Field(None, description="""The notes of the query""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + None -class Output(TestEntity): +class StandardsComplianceTestSuite(TestSuite): """ - Represents an output from a TestCase + Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model. """ + test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") + test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") + test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") + test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") -class SemanticSmokeTestOutput(Output): +class OneHopTestSuite(TestSuite): """ - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness. """ + test_metadata: Optional[str] = Field(None, description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") + test_persona: Optional[TestPersonaEnum] = Field(None, description="""A Test persona describes the user or operational context of a given test.""") + test_cases: Optional[Dict[str, TestCase]] = Field(default_factory=dict, description="""List of explicitly enumerated Test Cases.""") + test_case_specification: Optional[str] = Field(None, description="""Declarative specification of a set of Test Cases generated elsewhere (i.e. within a Test Runner)""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") @@ -398,20 +454,19 @@ class Precondition(TestEntity): TestEntity.model_rebuild() TestMetadata.model_rebuild() TestAsset.model_rebuild() -TestAssetCollection.model_rebuild() -QueryAnswerPair.model_rebuild() +AcceptanceTestAsset.model_rebuild() TestEdgeData.model_rebuild() TestCase.model_rebuild() TestCaseSpecification.model_rebuild() -TestSuite.model_rebuild() AcceptanceTestCase.model_rebuild() +QuantitativeTestCase.model_rebuild() +ComplianceTestCase.model_rebuild() +KnowledgeGraphNavigationTestCase.model_rebuild() +OneHopTestCase.model_rebuild() +TestSuite.model_rebuild() AcceptanceTestSuite.model_rebuild() BenchmarkTestSuite.model_rebuild() StandardsComplianceTestSuite.model_rebuild() OneHopTestSuite.model_rebuild() -Input.model_rebuild() -SemanticSmokeTestInput.model_rebuild() -Output.model_rebuild() -SemanticSmokeTestOutput.model_rebuild() Precondition.model_rebuild() diff --git a/src/translator_testing_model/datamodel/translator_testing_model.py b/src/translator_testing_model/datamodel/translator_testing_model.py index 023694f..c49aad8 100644 --- a/src/translator_testing_model/datamodel/translator_testing_model.py +++ b/src/translator_testing_model/datamodel/translator_testing_model.py @@ -1,5 +1,5 @@ # Auto generated from translator_testing_model.yaml by pythongen.py version: 0.0.1 -# Generation date: 2023-10-25T17:15:01 +# Generation date: 2023-10-26T21:50:42 # Schema: Translator-Testing-Model # # id: https://w3id.org/TranslatorSRI/TranslatorTestingModel @@ -22,8 +22,8 @@ from linkml_runtime.utils.enumerations import EnumDefinitionImpl from rdflib import Namespace, URIRef from linkml_runtime.utils.curienamespace import CurieNamespace -from linkml_runtime.linkml_model.types import Boolean, Curie, Date, String, Uriorcurie -from linkml_runtime.utils.metamodelcore import Bool, Curie, URIorCURIE, XSDDate +from linkml_runtime.linkml_model.types import Boolean, Date, Integer, String, Uriorcurie +from linkml_runtime.utils.metamodelcore import Bool, URIorCURIE, XSDDate metamodel_version = "1.7.0" version = "0.0.0" @@ -55,11 +55,7 @@ class TestAssetId(TestEntityId): pass -class TestAssetCollectionId(TestEntityId): - pass - - -class QueryAnswerPairId(TestAssetId): +class AcceptanceTestAssetId(TestAssetId): pass @@ -75,39 +71,39 @@ class TestCaseSpecificationId(TestEntityId): pass -class TestSuiteId(TestEntityId): +class AcceptanceTestCaseId(TestCaseId): pass -class AcceptanceTestCaseId(TestCaseId): +class QuantitativeTestCaseId(TestCaseId): pass -class AcceptanceTestSuiteId(TestSuiteId): +class ComplianceTestCaseId(TestCaseId): pass -class StandardsComplianceTestSuiteId(TestSuiteId): +class KnowledgeGraphNavigationTestCaseId(TestCaseId): pass -class OneHopTestSuiteId(TestSuiteId): +class OneHopTestCaseId(KnowledgeGraphNavigationTestCaseId): pass -class InputId(TestEntityId): +class TestSuiteId(TestEntityId): pass -class SemanticSmokeTestInputId(InputId): +class AcceptanceTestSuiteId(TestSuiteId): pass -class OutputId(TestEntityId): +class StandardsComplianceTestSuiteId(TestSuiteId): pass -class SemanticSmokeTestOutputId(OutputId): +class OneHopTestSuiteId(TestSuiteId): pass @@ -204,6 +200,7 @@ class TestAsset(TestEntity): id: Union[str, TestAssetId] = None input_id: Optional[Union[str, URIorCURIE]] = None input_name: Optional[str] = None + predicate: Optional[str] = None output_id: Optional[Union[str, URIorCURIE]] = None output_name: Optional[str] = None expected_output: Optional[Union[str, "ExpectedOutputEnum"]] = None @@ -225,6 +222,9 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self.input_name is not None and not isinstance(self.input_name, str): self.input_name = str(self.input_name) + if self.predicate is not None and not isinstance(self.predicate, str): + self.predicate = str(self.predicate) + if self.output_id is not None and not isinstance(self.output_id, URIorCURIE): self.output_id = URIorCURIE(self.output_id) @@ -254,57 +254,64 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): @dataclass -class TestAssetCollection(TestEntity): +class AcceptanceTestAsset(TestAsset): """ - Represents an ad hoc list of Test Assets. + Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. """ _inherited_slots: ClassVar[List[str]] = [] - class_class_uri: ClassVar[URIRef] = TTM.TestAssetCollection - class_class_curie: ClassVar[str] = "ttm:TestAssetCollection" - class_name: ClassVar[str] = "TestAssetCollection" - class_model_uri: ClassVar[URIRef] = TTM.TestAssetCollection + class_class_uri: ClassVar[URIRef] = TTM.AcceptanceTestAsset + class_class_curie: ClassVar[str] = "ttm:AcceptanceTestAsset" + class_name: ClassVar[str] = "AcceptanceTestAsset" + class_model_uri: ClassVar[URIRef] = TTM.AcceptanceTestAsset - id: Union[str, TestAssetCollectionId] = None - test_assets: Union[Dict[Union[str, TestCaseId], Union[dict, "TestCase"]], List[Union[dict, "TestCase"]]] = empty_dict() - tags: Optional[Union[str, List[str]]] = empty_list() + id: Union[str, AcceptanceTestAssetId] = None + must_pass_date: Optional[Union[str, XSDDate]] = None + must_pass_environment: Optional[Union[str, "TestEnvEnum"]] = None + query: Optional[str] = None + string_entry: Optional[str] = None + direction: Optional[Union[str, "DirectionEnum"]] = None + answer_informal_concept: Optional[str] = None + expected_result: Optional[Union[str, "ExpectedResultsEnum"]] = None + top_level: Optional[int] = None + query_node: Optional[Union[str, "NodeEnum"]] = None + notes: Optional[str] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): self.MissingRequiredField("id") - if not isinstance(self.id, TestAssetCollectionId): - self.id = TestAssetCollectionId(self.id) + if not isinstance(self.id, AcceptanceTestAssetId): + self.id = AcceptanceTestAssetId(self.id) - if self._is_empty(self.test_assets): - self.MissingRequiredField("test_assets") - self._normalize_inlined_as_dict(slot_name="test_assets", slot_type=TestCase, key_name="id", keyed=True) + if self.must_pass_date is not None and not isinstance(self.must_pass_date, XSDDate): + self.must_pass_date = XSDDate(self.must_pass_date) - if not isinstance(self.tags, list): - self.tags = [self.tags] if self.tags is not None else [] - self.tags = [v if isinstance(v, str) else str(v) for v in self.tags] + if self.must_pass_environment is not None and not isinstance(self.must_pass_environment, TestEnvEnum): + self.must_pass_environment = TestEnvEnum(self.must_pass_environment) - super().__post_init__(**kwargs) + if self.query is not None and not isinstance(self.query, str): + self.query = str(self.query) + if self.string_entry is not None and not isinstance(self.string_entry, str): + self.string_entry = str(self.string_entry) -@dataclass -class QueryAnswerPair(TestAsset): - """ - Represents a QueryAnswerPair specification of a Test Asset - """ - _inherited_slots: ClassVar[List[str]] = [] + if self.direction is not None and not isinstance(self.direction, DirectionEnum): + self.direction = DirectionEnum(self.direction) - class_class_uri: ClassVar[URIRef] = TTM.QueryAnswerPair - class_class_curie: ClassVar[str] = "ttm:QueryAnswerPair" - class_name: ClassVar[str] = "QueryAnswerPair" - class_model_uri: ClassVar[URIRef] = TTM.QueryAnswerPair + if self.answer_informal_concept is not None and not isinstance(self.answer_informal_concept, str): + self.answer_informal_concept = str(self.answer_informal_concept) - id: Union[str, QueryAnswerPairId] = None + if self.expected_result is not None and not isinstance(self.expected_result, ExpectedResultsEnum): + self.expected_result = ExpectedResultsEnum(self.expected_result) - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.id): - self.MissingRequiredField("id") - if not isinstance(self.id, QueryAnswerPairId): - self.id = QueryAnswerPairId(self.id) + if self.top_level is not None and not isinstance(self.top_level, int): + self.top_level = int(self.top_level) + + if self.query_node is not None and not isinstance(self.query_node, NodeEnum): + self.query_node = NodeEnum(self.query_node) + + if self.notes is not None and not isinstance(self.notes, str): + self.notes = str(self.notes) super().__post_init__(**kwargs) @@ -312,7 +319,8 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): @dataclass class TestEdgeData(TestAsset): """ - Represents a single Biolink Model compliant instance of an edge that can be used for testing. + Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for + testing. """ _inherited_slots: ClassVar[List[str]] = [] @@ -335,8 +343,9 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): @dataclass class TestCase(TestEntity): """ - Represents a single enumerated instance of Test Case, derived from a given TestAsset and used to probe a - particular test condition. + Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset + instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to + probe a particular test condition. """ _inherited_slots: ClassVar[List[str]] = [] @@ -346,9 +355,12 @@ class TestCase(TestEntity): class_model_uri: ClassVar[URIRef] = TTM.TestCase id: Union[str, TestCaseId] = None - inputs: Optional[Union[Union[str, InputId], List[Union[str, InputId]]]] = empty_list() - outputs: Optional[Union[Union[str, OutputId], List[Union[str, OutputId]]]] = empty_list() + test_assets: Union[Dict[Union[str, TestCaseId], Union[dict, "TestCase"]], List[Union[dict, "TestCase"]]] = empty_dict() + test_env: Optional[Union[str, "TestEnvEnum"]] = None + test_case_type: Optional[Union[str, "TestCaseTypeEnum"]] = None + query_type: Optional[Union[str, "QueryTypeEnum"]] = None preconditions: Optional[Union[Union[str, PreconditionId], List[Union[str, PreconditionId]]]] = empty_list() + tags: Optional[Union[str, List[str]]] = empty_list() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): @@ -356,18 +368,27 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if not isinstance(self.id, TestCaseId): self.id = TestCaseId(self.id) - if not isinstance(self.inputs, list): - self.inputs = [self.inputs] if self.inputs is not None else [] - self.inputs = [v if isinstance(v, InputId) else InputId(v) for v in self.inputs] + if self._is_empty(self.test_assets): + self.MissingRequiredField("test_assets") + self._normalize_inlined_as_list(slot_name="test_assets", slot_type=TestCase, key_name="id", keyed=True) + + if self.test_env is not None and not isinstance(self.test_env, TestEnvEnum): + self.test_env = TestEnvEnum(self.test_env) - if not isinstance(self.outputs, list): - self.outputs = [self.outputs] if self.outputs is not None else [] - self.outputs = [v if isinstance(v, OutputId) else OutputId(v) for v in self.outputs] + if self.test_case_type is not None and not isinstance(self.test_case_type, TestCaseTypeEnum): + self.test_case_type = TestCaseTypeEnum(self.test_case_type) + + if self.query_type is not None and not isinstance(self.query_type, QueryTypeEnum): + self.query_type = QueryTypeEnum(self.query_type) if not isinstance(self.preconditions, list): self.preconditions = [self.preconditions] if self.preconditions is not None else [] self.preconditions = [v if isinstance(v, PreconditionId) else PreconditionId(v) for v in self.preconditions] + if not isinstance(self.tags, list): + self.tags = [self.tags] if self.tags is not None else [] + self.tags = [v if isinstance(v, str) else str(v) for v in self.tags] + super().__post_init__(**kwargs) @@ -395,50 +416,10 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): super().__post_init__(**kwargs) -@dataclass -class TestSuite(TestEntity): - """ - Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic - 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be - present. - """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = TTM.TestSuite - class_class_curie: ClassVar[str] = "ttm:TestSuite" - class_name: ClassVar[str] = "TestSuite" - class_model_uri: ClassVar[URIRef] = TTM.TestSuite - - id: Union[str, TestSuiteId] = None - test_metadata: Optional[Union[str, TestMetadataId]] = None - test_persona: Optional[Union[str, "TestPersonaEnum"]] = None - test_cases: Optional[Union[Dict[Union[str, TestCaseId], Union[dict, TestCase]], List[Union[dict, TestCase]]]] = empty_dict() - test_case_specification: Optional[Union[str, TestCaseSpecificationId]] = None - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.id): - self.MissingRequiredField("id") - if not isinstance(self.id, TestSuiteId): - self.id = TestSuiteId(self.id) - - if self.test_metadata is not None and not isinstance(self.test_metadata, TestMetadataId): - self.test_metadata = TestMetadataId(self.test_metadata) - - if self.test_persona is not None and not isinstance(self.test_persona, TestPersonaEnum): - self.test_persona = TestPersonaEnum(self.test_persona) - - self._normalize_inlined_as_dict(slot_name="test_cases", slot_type=TestCase, key_name="id", keyed=True) - - if self.test_case_specification is not None and not isinstance(self.test_case_specification, TestCaseSpecificationId): - self.test_case_specification = TestCaseSpecificationId(self.test_case_specification) - - super().__post_init__(**kwargs) - - @dataclass class AcceptanceTestCase(TestCase): """ - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + See AcceptanceTestAsset above for more details. """ _inherited_slots: ClassVar[List[str]] = [] @@ -448,8 +429,7 @@ class AcceptanceTestCase(TestCase): class_model_uri: ClassVar[URIRef] = TTM.AcceptanceTestCase id: Union[str, AcceptanceTestCaseId] = None - inputs: Union[Union[str, SemanticSmokeTestInputId], List[Union[str, SemanticSmokeTestInputId]]] = None - outputs: Union[Union[str, SemanticSmokeTestOutputId], List[Union[str, SemanticSmokeTestOutputId]]] = None + test_assets: Union[Dict[Union[str, AcceptanceTestAssetId], Union[dict, AcceptanceTestAsset]], List[Union[dict, AcceptanceTestAsset]]] = empty_dict() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): @@ -457,231 +437,223 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if not isinstance(self.id, AcceptanceTestCaseId): self.id = AcceptanceTestCaseId(self.id) - if self._is_empty(self.inputs): - self.MissingRequiredField("inputs") - if not isinstance(self.inputs, list): - self.inputs = [self.inputs] if self.inputs is not None else [] - self.inputs = [v if isinstance(v, SemanticSmokeTestInputId) else SemanticSmokeTestInputId(v) for v in self.inputs] - - if self._is_empty(self.outputs): - self.MissingRequiredField("outputs") - if not isinstance(self.outputs, list): - self.outputs = [self.outputs] if self.outputs is not None else [] - self.outputs = [v if isinstance(v, SemanticSmokeTestOutputId) else SemanticSmokeTestOutputId(v) for v in self.outputs] + if self._is_empty(self.test_assets): + self.MissingRequiredField("test_assets") + self._normalize_inlined_as_list(slot_name="test_assets", slot_type=AcceptanceTestAsset, key_name="id", keyed=True) super().__post_init__(**kwargs) @dataclass -class AcceptanceTestSuite(TestSuite): +class QuantitativeTestCase(TestCase): + """ + Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + """ _inherited_slots: ClassVar[List[str]] = [] - class_class_uri: ClassVar[URIRef] = TTM.AcceptanceTestSuite - class_class_curie: ClassVar[str] = "ttm:AcceptanceTestSuite" - class_name: ClassVar[str] = "AcceptanceTestSuite" - class_model_uri: ClassVar[URIRef] = TTM.AcceptanceTestSuite + class_class_uri: ClassVar[URIRef] = TTM.QuantitativeTestCase + class_class_curie: ClassVar[str] = "ttm:QuantitativeTestCase" + class_name: ClassVar[str] = "QuantitativeTestCase" + class_model_uri: ClassVar[URIRef] = TTM.QuantitativeTestCase - id: Union[str, AcceptanceTestSuiteId] = None + id: Union[str, QuantitativeTestCaseId] = None + test_assets: Union[Dict[Union[str, TestCaseId], Union[dict, TestCase]], List[Union[dict, TestCase]]] = empty_dict() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): self.MissingRequiredField("id") - if not isinstance(self.id, AcceptanceTestSuiteId): - self.id = AcceptanceTestSuiteId(self.id) + if not isinstance(self.id, QuantitativeTestCaseId): + self.id = QuantitativeTestCaseId(self.id) super().__post_init__(**kwargs) -class BenchmarkTestSuite(YAMLRoot): - """ - JsonObj(is_a='TestSuite') - """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = TTM.BenchmarkTestSuite - class_class_curie: ClassVar[str] = "ttm:BenchmarkTestSuite" - class_name: ClassVar[str] = "BenchmarkTestSuite" - class_model_uri: ClassVar[URIRef] = TTM.BenchmarkTestSuite - - @dataclass -class StandardsComplianceTestSuite(TestSuite): +class ComplianceTestCase(TestCase): """ - Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model. + TRAPI and Biolink Model standards compliance test """ _inherited_slots: ClassVar[List[str]] = [] - class_class_uri: ClassVar[URIRef] = TTM.StandardsComplianceTestSuite - class_class_curie: ClassVar[str] = "ttm:StandardsComplianceTestSuite" - class_name: ClassVar[str] = "StandardsComplianceTestSuite" - class_model_uri: ClassVar[URIRef] = TTM.StandardsComplianceTestSuite + class_class_uri: ClassVar[URIRef] = TTM.ComplianceTestCase + class_class_curie: ClassVar[str] = "ttm:ComplianceTestCase" + class_name: ClassVar[str] = "ComplianceTestCase" + class_model_uri: ClassVar[URIRef] = TTM.ComplianceTestCase - id: Union[str, StandardsComplianceTestSuiteId] = None + id: Union[str, ComplianceTestCaseId] = None + test_assets: Union[Dict[Union[str, TestCaseId], Union[dict, TestCase]], List[Union[dict, TestCase]]] = empty_dict() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): self.MissingRequiredField("id") - if not isinstance(self.id, StandardsComplianceTestSuiteId): - self.id = StandardsComplianceTestSuiteId(self.id) + if not isinstance(self.id, ComplianceTestCaseId): + self.id = ComplianceTestCaseId(self.id) super().__post_init__(**kwargs) @dataclass -class OneHopTestSuite(TestSuite): +class KnowledgeGraphNavigationTestCase(TestCase): """ - Test case for testing the integrity of "One Hop" knowledge graph retrievals sensa legacy SRI_Testing harness. + Knowledge Graph navigation integration test """ _inherited_slots: ClassVar[List[str]] = [] - class_class_uri: ClassVar[URIRef] = TTM.OneHopTestSuite - class_class_curie: ClassVar[str] = "ttm:OneHopTestSuite" - class_name: ClassVar[str] = "OneHopTestSuite" - class_model_uri: ClassVar[URIRef] = TTM.OneHopTestSuite + class_class_uri: ClassVar[URIRef] = TTM.KnowledgeGraphNavigationTestCase + class_class_curie: ClassVar[str] = "ttm:KnowledgeGraphNavigationTestCase" + class_name: ClassVar[str] = "KnowledgeGraphNavigationTestCase" + class_model_uri: ClassVar[URIRef] = TTM.KnowledgeGraphNavigationTestCase - id: Union[str, OneHopTestSuiteId] = None + id: Union[str, KnowledgeGraphNavigationTestCaseId] = None + test_assets: Union[Dict[Union[str, TestCaseId], Union[dict, TestCase]], List[Union[dict, TestCase]]] = empty_dict() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): self.MissingRequiredField("id") - if not isinstance(self.id, OneHopTestSuiteId): - self.id = OneHopTestSuiteId(self.id) + if not isinstance(self.id, KnowledgeGraphNavigationTestCaseId): + self.id = KnowledgeGraphNavigationTestCaseId(self.id) super().__post_init__(**kwargs) @dataclass -class Input(TestEntity): +class OneHopTestCase(KnowledgeGraphNavigationTestCase): """ - Represents an input to a TestCase + 'One Hop' Knowledge Graph navigation integration test """ _inherited_slots: ClassVar[List[str]] = [] - class_class_uri: ClassVar[URIRef] = TTM.Input - class_class_curie: ClassVar[str] = "ttm:Input" - class_name: ClassVar[str] = "Input" - class_model_uri: ClassVar[URIRef] = TTM.Input + class_class_uri: ClassVar[URIRef] = TTM.OneHopTestCase + class_class_curie: ClassVar[str] = "ttm:OneHopTestCase" + class_name: ClassVar[str] = "OneHopTestCase" + class_model_uri: ClassVar[URIRef] = TTM.OneHopTestCase - id: Union[str, InputId] = None + id: Union[str, OneHopTestCaseId] = None + test_assets: Union[Dict[Union[str, TestCaseId], Union[dict, TestCase]], List[Union[dict, TestCase]]] = empty_dict() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): self.MissingRequiredField("id") - if not isinstance(self.id, InputId): - self.id = InputId(self.id) + if not isinstance(self.id, OneHopTestCaseId): + self.id = OneHopTestCaseId(self.id) super().__post_init__(**kwargs) @dataclass -class SemanticSmokeTestInput(Input): +class TestSuite(TestEntity): """ - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic + 'test_case_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be + present. """ _inherited_slots: ClassVar[List[str]] = [] - class_class_uri: ClassVar[URIRef] = TTM.SemanticSmokeTestInput - class_class_curie: ClassVar[str] = "ttm:SemanticSmokeTestInput" - class_name: ClassVar[str] = "SemanticSmokeTestInput" - class_model_uri: ClassVar[URIRef] = TTM.SemanticSmokeTestInput + class_class_uri: ClassVar[URIRef] = TTM.TestSuite + class_class_curie: ClassVar[str] = "ttm:TestSuite" + class_name: ClassVar[str] = "TestSuite" + class_model_uri: ClassVar[URIRef] = TTM.TestSuite - id: Union[str, SemanticSmokeTestInputId] = None - must_pass_date: Optional[Union[str, XSDDate]] = None - must_pass_environment: Optional[Union[str, "EnvironmentEnum"]] = None - query: Optional[str] = None - string_entry: Optional[str] = None - direction: Optional[Union[str, "DirectionEnum"]] = None - answer_informal_concept: Optional[str] = None - expected_result: Optional[Union[str, "ExpectedResultsEnum"]] = None - curie: Optional[Union[str, Curie]] = None - top_level: Optional[str] = None - node: Optional[str] = None - notes: Optional[str] = None + id: Union[str, TestSuiteId] = None + test_metadata: Optional[Union[str, TestMetadataId]] = None + test_persona: Optional[Union[str, "TestPersonaEnum"]] = None + test_cases: Optional[Union[Dict[Union[str, TestCaseId], Union[dict, TestCase]], List[Union[dict, TestCase]]]] = empty_dict() + test_case_specification: Optional[Union[str, TestCaseSpecificationId]] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): self.MissingRequiredField("id") - if not isinstance(self.id, SemanticSmokeTestInputId): - self.id = SemanticSmokeTestInputId(self.id) - - if self.must_pass_date is not None and not isinstance(self.must_pass_date, XSDDate): - self.must_pass_date = XSDDate(self.must_pass_date) + if not isinstance(self.id, TestSuiteId): + self.id = TestSuiteId(self.id) - if self.must_pass_environment is not None and not isinstance(self.must_pass_environment, EnvironmentEnum): - self.must_pass_environment = EnvironmentEnum(self.must_pass_environment) + if self.test_metadata is not None and not isinstance(self.test_metadata, TestMetadataId): + self.test_metadata = TestMetadataId(self.test_metadata) - if self.query is not None and not isinstance(self.query, str): - self.query = str(self.query) + if self.test_persona is not None and not isinstance(self.test_persona, TestPersonaEnum): + self.test_persona = TestPersonaEnum(self.test_persona) - if self.string_entry is not None and not isinstance(self.string_entry, str): - self.string_entry = str(self.string_entry) + self._normalize_inlined_as_dict(slot_name="test_cases", slot_type=TestCase, key_name="id", keyed=True) - if self.direction is not None and not isinstance(self.direction, DirectionEnum): - self.direction = DirectionEnum(self.direction) + if self.test_case_specification is not None and not isinstance(self.test_case_specification, TestCaseSpecificationId): + self.test_case_specification = TestCaseSpecificationId(self.test_case_specification) - if self.answer_informal_concept is not None and not isinstance(self.answer_informal_concept, str): - self.answer_informal_concept = str(self.answer_informal_concept) + super().__post_init__(**kwargs) - if self.expected_result is not None and not isinstance(self.expected_result, ExpectedResultsEnum): - self.expected_result = ExpectedResultsEnum(self.expected_result) - if self.curie is not None and not isinstance(self.curie, Curie): - self.curie = Curie(self.curie) +@dataclass +class AcceptanceTestSuite(TestSuite): + _inherited_slots: ClassVar[List[str]] = [] - if self.top_level is not None and not isinstance(self.top_level, str): - self.top_level = str(self.top_level) + class_class_uri: ClassVar[URIRef] = TTM.AcceptanceTestSuite + class_class_curie: ClassVar[str] = "ttm:AcceptanceTestSuite" + class_name: ClassVar[str] = "AcceptanceTestSuite" + class_model_uri: ClassVar[URIRef] = TTM.AcceptanceTestSuite - if self.node is not None and not isinstance(self.node, str): - self.node = str(self.node) + id: Union[str, AcceptanceTestSuiteId] = None - if self.notes is not None and not isinstance(self.notes, str): - self.notes = str(self.notes) + def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): + if self._is_empty(self.id): + self.MissingRequiredField("id") + if not isinstance(self.id, AcceptanceTestSuiteId): + self.id = AcceptanceTestSuiteId(self.id) super().__post_init__(**kwargs) +class BenchmarkTestSuite(YAMLRoot): + """ + JsonObj(is_a='TestSuite') + """ + _inherited_slots: ClassVar[List[str]] = [] + + class_class_uri: ClassVar[URIRef] = TTM.BenchmarkTestSuite + class_class_curie: ClassVar[str] = "ttm:BenchmarkTestSuite" + class_name: ClassVar[str] = "BenchmarkTestSuite" + class_model_uri: ClassVar[URIRef] = TTM.BenchmarkTestSuite + + @dataclass -class Output(TestEntity): +class StandardsComplianceTestSuite(TestSuite): """ - Represents an output from a TestCase + Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model. """ _inherited_slots: ClassVar[List[str]] = [] - class_class_uri: ClassVar[URIRef] = TTM.Output - class_class_curie: ClassVar[str] = "ttm:Output" - class_name: ClassVar[str] = "Output" - class_model_uri: ClassVar[URIRef] = TTM.Output + class_class_uri: ClassVar[URIRef] = TTM.StandardsComplianceTestSuite + class_class_curie: ClassVar[str] = "ttm:StandardsComplianceTestSuite" + class_name: ClassVar[str] = "StandardsComplianceTestSuite" + class_model_uri: ClassVar[URIRef] = TTM.StandardsComplianceTestSuite - id: Union[str, OutputId] = None + id: Union[str, StandardsComplianceTestSuiteId] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): self.MissingRequiredField("id") - if not isinstance(self.id, OutputId): - self.id = OutputId(self.id) + if not isinstance(self.id, StandardsComplianceTestSuiteId): + self.id = StandardsComplianceTestSuiteId(self.id) super().__post_init__(**kwargs) @dataclass -class SemanticSmokeTestOutput(Output): +class OneHopTestSuite(TestSuite): """ - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + Test case for testing the integrity of "One Hop" knowledge graph retrievals sensa legacy SRI_Testing harness. """ _inherited_slots: ClassVar[List[str]] = [] - class_class_uri: ClassVar[URIRef] = TTM.SemanticSmokeTestOutput - class_class_curie: ClassVar[str] = "ttm:SemanticSmokeTestOutput" - class_name: ClassVar[str] = "SemanticSmokeTestOutput" - class_model_uri: ClassVar[URIRef] = TTM.SemanticSmokeTestOutput + class_class_uri: ClassVar[URIRef] = TTM.OneHopTestSuite + class_class_curie: ClassVar[str] = "ttm:OneHopTestSuite" + class_name: ClassVar[str] = "OneHopTestSuite" + class_model_uri: ClassVar[URIRef] = TTM.OneHopTestSuite - id: Union[str, SemanticSmokeTestOutputId] = None + id: Union[str, OneHopTestSuiteId] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): self.MissingRequiredField("id") - if not isinstance(self.id, SemanticSmokeTestOutputId): - self.id = SemanticSmokeTestOutputId(self.id) + if not isinstance(self.id, OneHopTestSuiteId): + self.id = OneHopTestSuiteId(self.id) super().__post_init__(**kwargs) @@ -771,11 +743,23 @@ class TestPersonaEnum(EnumDefinitionImpl): name="TestPersonaEnum", ) +class QueryTypeEnum(EnumDefinitionImpl): + """ + Query + """ + treats = PermissibleValue(text="treats") + + _defn = EnumDefinition( + name="QueryTypeEnum", + description="Query", + ) + class ExpectedOutputEnum(EnumDefinitionImpl): """ Expected output values for instances of Test Asset or Test Cases(?). (Note: does this Enum overlap with 'ExpectedResultsEnum' below?) """ + Top_Answer = PermissibleValue(text="Top_Answer") Acceptable = PermissibleValue(text="Acceptable") BadButForgivable = PermissibleValue(text="BadButForgivable") NeverShow = PermissibleValue(text="NeverShow") @@ -808,11 +792,70 @@ class ExpectedResultsEnum(EnumDefinitionImpl): description="Does this Enum overlap with 'ExpectedOutputEnum' above?", ) +class NodeEnum(EnumDefinitionImpl): + """ + Target node of a Subject-Predicate-Object driven query + """ + subject = PermissibleValue(text="subject") + object = PermissibleValue(text="object") + + _defn = EnumDefinition( + name="NodeEnum", + description="Target node of a Subject-Predicate-Object driven query", + ) + +class TestEnvEnum(EnumDefinitionImpl): + """ + Testing environments within which a TestSuite is run by a TestRunner scheduled by the TestHarness. + """ + dev = PermissibleValue( + text="dev", + description="Development") + ci = PermissibleValue( + text="ci", + description="Continuous Integration") + test = PermissibleValue( + text="test", + description="Test") + prod = PermissibleValue( + text="prod", + description="Production") + + _defn = EnumDefinition( + name="TestEnvEnum", + description="Testing environments within which a TestSuite is run by a TestRunner scheduled by the TestHarness.", + ) + +class TestCaseTypeEnum(EnumDefinitionImpl): + """ + Enumerated tags for types of test (generally applied to a TestCase). + """ + acceptance = PermissibleValue( + text="acceptance", + description="Acceptance test") + quantitative = PermissibleValue( + text="quantitative", + description="Quantitative test") + compliance = PermissibleValue( + text="compliance", + description="Standards compliance test") + kg_navigation = PermissibleValue( + text="kg_navigation", + description="Knowledge Graph navigation integration test") + one_hop = PermissibleValue( + text="one_hop", + description="One Hop navigation test") + + _defn = EnumDefinition( + name="TestCaseTypeEnum", + description="Enumerated tags for types of test (generally applied to a TestCase).", + ) + class TestIssueEnum(EnumDefinitionImpl): TMKP = PermissibleValue( text="TMKP", - description="Text Mining Knowledge Provider generated relationship?") + description="'Text Mining Knowledge Provider' generated relationship?") contraindications = PermissibleValue(text="contraindications") _defn = EnumDefinition( @@ -850,17 +893,6 @@ class DirectionEnum(EnumDefinitionImpl): name="DirectionEnum", ) -class EnvironmentEnum(EnumDefinitionImpl): - - DEV = PermissibleValue(text="DEV") - CI = PermissibleValue(text="CI") - TEST = PermissibleValue(text="TEST") - PROD = PermissibleValue(text="PROD") - - _defn = EnumDefinition( - name="EnvironmentEnum", - ) - # Slots class slots: pass @@ -892,6 +924,9 @@ class slots: slots.input_name = Slot(uri=TTM.input_name, name="input_name", curie=TTM.curie('input_name'), model_uri=TTM.input_name, domain=None, range=Optional[str]) +slots.predicate = Slot(uri=TTM.predicate, name="predicate", curie=TTM.curie('predicate'), + model_uri=TTM.predicate, domain=None, range=Optional[str]) + slots.output_id = Slot(uri=TTM.output_id, name="output_id", curie=TTM.curie('output_id'), model_uri=TTM.output_id, domain=None, range=Optional[Union[str, URIorCURIE]]) @@ -932,7 +967,7 @@ class slots: model_uri=TTM.must_pass_date, domain=None, range=Optional[Union[str, XSDDate]]) slots.must_pass_environment = Slot(uri=TTM.must_pass_environment, name="must_pass_environment", curie=TTM.curie('must_pass_environment'), - model_uri=TTM.must_pass_environment, domain=None, range=Optional[Union[str, "EnvironmentEnum"]]) + model_uri=TTM.must_pass_environment, domain=None, range=Optional[Union[str, "TestEnvEnum"]]) slots.query = Slot(uri=TTM.query, name="query", curie=TTM.curie('query'), model_uri=TTM.query, domain=None, range=Optional[str]) @@ -949,14 +984,11 @@ class slots: slots.expected_result = Slot(uri=TTM.expected_result, name="expected_result", curie=TTM.curie('expected_result'), model_uri=TTM.expected_result, domain=None, range=Optional[Union[str, "ExpectedResultsEnum"]]) -slots.curie = Slot(uri=TTM.curie, name="curie", curie=TTM.curie('curie'), - model_uri=TTM.curie, domain=None, range=Optional[Union[str, Curie]]) - slots.top_level = Slot(uri=TTM.top_level, name="top_level", curie=TTM.curie('top_level'), - model_uri=TTM.top_level, domain=None, range=Optional[str]) + model_uri=TTM.top_level, domain=None, range=Optional[int]) -slots.node = Slot(uri=TTM.node, name="node", curie=TTM.curie('node'), - model_uri=TTM.node, domain=None, range=Optional[str]) +slots.query_node = Slot(uri=TTM.query_node, name="query_node", curie=TTM.curie('query_node'), + model_uri=TTM.query_node, domain=None, range=Optional[Union[str, "NodeEnum"]]) slots.notes = Slot(uri=TTM.notes, name="notes", curie=TTM.curie('notes'), model_uri=TTM.notes, domain=None, range=Optional[str]) @@ -964,11 +996,14 @@ class slots: slots.requires_trapi = Slot(uri=TTM.requires_trapi, name="requires_trapi", curie=TTM.curie('requires_trapi'), model_uri=TTM.requires_trapi, domain=None, range=Optional[Union[bool, Bool]]) -slots.inputs = Slot(uri=TTM.inputs, name="inputs", curie=TTM.curie('inputs'), - model_uri=TTM.inputs, domain=None, range=Optional[Union[Union[str, InputId], List[Union[str, InputId]]]]) +slots.test_env = Slot(uri=TTM.test_env, name="test_env", curie=TTM.curie('test_env'), + model_uri=TTM.test_env, domain=None, range=Optional[Union[str, "TestEnvEnum"]]) -slots.outputs = Slot(uri=TTM.outputs, name="outputs", curie=TTM.curie('outputs'), - model_uri=TTM.outputs, domain=None, range=Optional[Union[Union[str, OutputId], List[Union[str, OutputId]]]]) +slots.test_case_type = Slot(uri=TTM.test_case_type, name="test_case_type", curie=TTM.curie('test_case_type'), + model_uri=TTM.test_case_type, domain=None, range=Optional[Union[str, "TestCaseTypeEnum"]]) + +slots.query_type = Slot(uri=TTM.query_type, name="query_type", curie=TTM.curie('query_type'), + model_uri=TTM.query_type, domain=None, range=Optional[Union[str, "QueryTypeEnum"]]) slots.preconditions = Slot(uri=TTM.preconditions, name="preconditions", curie=TTM.curie('preconditions'), model_uri=TTM.preconditions, domain=None, range=Optional[Union[Union[str, PreconditionId], List[Union[str, PreconditionId]]]]) @@ -979,11 +1014,14 @@ class slots: slots.TestAsset_tags = Slot(uri=SCHEMA.additionalType, name="TestAsset_tags", curie=SCHEMA.curie('additionalType'), model_uri=TTM.TestAsset_tags, domain=TestAsset, range=Optional[Union[str, List[str]]]) -slots.TestAssetCollection_tags = Slot(uri=SCHEMA.additionalType, name="TestAssetCollection_tags", curie=SCHEMA.curie('additionalType'), - model_uri=TTM.TestAssetCollection_tags, domain=TestAssetCollection, range=Optional[Union[str, List[str]]]) +slots.TestCase_test_case_type = Slot(uri=TTM.test_case_type, name="TestCase_test_case_type", curie=TTM.curie('test_case_type'), + model_uri=TTM.TestCase_test_case_type, domain=TestCase, range=Optional[Union[str, "TestCaseTypeEnum"]]) + +slots.TestCase_test_assets = Slot(uri=TTM.test_assets, name="TestCase_test_assets", curie=TTM.curie('test_assets'), + model_uri=TTM.TestCase_test_assets, domain=TestCase, range=Union[Dict[Union[str, TestCaseId], Union[dict, "TestCase"]], List[Union[dict, "TestCase"]]]) -slots.AcceptanceTestCase_inputs = Slot(uri=TTM.inputs, name="AcceptanceTestCase_inputs", curie=TTM.curie('inputs'), - model_uri=TTM.AcceptanceTestCase_inputs, domain=AcceptanceTestCase, range=Union[Union[str, SemanticSmokeTestInputId], List[Union[str, SemanticSmokeTestInputId]]]) +slots.TestCase_tags = Slot(uri=SCHEMA.additionalType, name="TestCase_tags", curie=SCHEMA.curie('additionalType'), + model_uri=TTM.TestCase_tags, domain=TestCase, range=Optional[Union[str, List[str]]]) -slots.AcceptanceTestCase_outputs = Slot(uri=TTM.outputs, name="AcceptanceTestCase_outputs", curie=TTM.curie('outputs'), - model_uri=TTM.AcceptanceTestCase_outputs, domain=AcceptanceTestCase, range=Union[Union[str, SemanticSmokeTestOutputId], List[Union[str, SemanticSmokeTestOutputId]]]) \ No newline at end of file +slots.AcceptanceTestCase_test_assets = Slot(uri=TTM.test_assets, name="AcceptanceTestCase_test_assets", curie=TTM.curie('test_assets'), + model_uri=TTM.AcceptanceTestCase_test_assets, domain=AcceptanceTestCase, range=Union[Dict[Union[str, AcceptanceTestAssetId], Union[dict, AcceptanceTestAsset]], List[Union[dict, AcceptanceTestAsset]]]) \ No newline at end of file diff --git a/src/translator_testing_model/schema/translator_testing_model.yaml b/src/translator_testing_model/schema/translator_testing_model.yaml index c292ccc..ec51168 100644 --- a/src/translator_testing_model/schema/translator_testing_model.yaml +++ b/src/translator_testing_model/schema/translator_testing_model.yaml @@ -56,6 +56,7 @@ classes: slots: - input_id - input_name + - predicate - output_id - output_name - expected_output @@ -72,40 +73,65 @@ classes: One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a "Block List" collection - TestAssetCollection: + AcceptanceTestAsset: + is_a: TestAsset description: >- - Represents an ad hoc list of Test Assets. - aliases: ["BlockList"] - is_a: TestEntity + Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. slots: - - test_assets - - tags - slot_usage: - tags: - description: >- - One or more 'tags' slot values (inherited from TestEntity) should generally - be defined as filters to specify TestAsset membership in a "Block List" collection + - must_pass_date + - must_pass_environment + - query + - string_entry + - direction + - answer_informal_concept + - expected_result +# - curie # use input_id + - top_level + - query_node + - notes - QueryAnswerPair: - description: >- - Represents a QueryAnswerPair specification of a Test Asset - # aliases: [""] - is_a: TestAsset +# QueryAnswerPair: +# description: >- +# Represents a QueryAnswerPair specification type of Test Asset. +# (is this class really needed? Let's assume that the nothing +# is added above the other test types) +# # aliases: [""] +# is_a: TestAsset TestEdgeData: description: >- - Represents a single Biolink Model compliant instance of an edge that can be used for testing. + Represents a single Biolink Model compliant instance of a + subject-predicate-object edge that can be used for testing. is_a: TestAsset TestCase: description: >- Represents a single enumerated instance of Test Case, derived from a - given TestAsset and used to probe a particular test condition. + given collection of one or more TestAsset instances (the value of the + 'test_assets' slot) which define the 'inputs' and 'outputs' of the + TestCase, used to probe a particular test condition. is_a: TestEntity slots: - - inputs - - outputs + - test_env + - test_case_type + - query_type + - test_assets - preconditions + slot_usage: + test_case_type: + description: >- + Is this valid or even necessary with the class names already + available from classes of 'test_assets' used (seems redundant?) + test_assets: + aliases: ["Block List"] + description: >- + One or more 'tags' slot values (inherited from TestEntity) + should generally be defined as filters to specify TestAsset + membership in 'test_assets' slot ("Block List") collection. + tags: + description: >- + One or more 'tags' slot values (inherited from TestEntity) should generally + be defined as filters to specify TestAsset membership in a "Block List" collection. TestCaseSpecification: description: >- @@ -113,6 +139,46 @@ classes: generates a collection of Test Cases from Test Assets, using applicable heuristics. is_a: TestEntity + AcceptanceTestCase: + is_a: TestCase + description: >- + See AcceptanceTestAsset above for more details. + slot_usage: + test_assets: + range: AcceptanceTestAsset + + QuantitativeTestCase: + is_a: TestCase + description: >- + Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. + slot_usage: +# test_assets: +# range: QuantitativeTestAsset + + ComplianceTestCase: + is_a: TestCase + description: >- + TRAPI and Biolink Model standards compliance test + slot_usage: +# test_assets: +# range: ComplianceTestAsset + + KnowledgeGraphNavigationTestCase: + is_a: TestCase + description: >- + Knowledge Graph navigation integration test + slot_usage: +# test_assets: +# range: KnowledgeGraphNavigationTestAsset + + OneHopTestCase: + is_a: KnowledgeGraphNavigationTestCase + description: >- + 'One Hop' Knowledge Graph navigation integration test + slot_usage: +# test_assets: +# range: OneHopTestAsset + TestSuite: is_a: TestEntity description: >- @@ -125,18 +191,6 @@ classes: - test_cases - test_case_specification - AcceptanceTestCase: - is_a: TestCase - description: >- - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. - slot_usage: - inputs: - range: SemanticSmokeTestInput - required: true - outputs: - range: SemanticSmokeTestOutput - required: true - AcceptanceTestSuite: is_a: TestSuite description: @@ -155,40 +209,6 @@ classes: Test case for testing the integrity of "One Hop" knowledge graph retrievals sensa legacy SRI_Testing harness. is_a: TestSuite - Input: - description: >- - Represents an input to a TestCase - is_a: TestEntity - slots: - - SemanticSmokeTestInput: - is_a: Input - description: >- - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. - slots: - - must_pass_date - - must_pass_environment - - query - - string_entry - - direction - - answer_informal_concept - - expected_result - - curie - - top_level - - node - - notes - - Output: - description: >- - Represents an output from a TestCase - is_a: TestEntity - slots: - - SemanticSmokeTestOutput: - is_a: Output - description: >- - Lifting schema from Shervin's runner JSON here as an example. This schema is not yet complete. - Precondition: description: >- Represents a precondition for a TestCase @@ -241,6 +261,9 @@ slots: input_name: aliases: ["InputName (user choice)"] + predicate: + aliases: ["Query"] + output_id: aliases: ["OutputID"] range: uriorcurie @@ -280,10 +303,15 @@ slots: test_assets: description: >- - List of explicitly enumerated Test Assets. + List of explicitly enumerated Test Assets. The class attributes of + TestAsset would be included in the TestCase versus being referred to by + the identifier (curie) of the TestAsset. That is, this would be a list of + objects (in JSONSchema serialization) versus a list of strings (where + each string is an identifier pointing to another class). range: TestCase multivalued: true inlined: true + inlined_as_list: true required: true test_cases: @@ -309,12 +337,12 @@ slots: must_pass_environment: description: >- - The environment in which this test must pass - range: EnvironmentEnum + The deployment environment within which this test must pass. + range: TestEnvEnum examples: - - value: "PROD" - - value: "CI" - - value: "TEST" + - value: "prod" + - value: "ci" + - value: "test" query: description: >- @@ -342,7 +370,8 @@ slots: description: >- An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of - strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0 + strictly sticking to the flat schema introduced by the spreadsheet here: + https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0 range: string examples: - value: "fingolimod" @@ -355,20 +384,21 @@ slots: The expected result of the query range: ExpectedResultsEnum - curie: - description: >- - The curie of the query - range: curie +# curie: +# description: >- +# The curie of the query; this is only used in AcceptanceTestCase +# which is now converted to an AcceptanceTestAsset which has 'input_id' +# range: curie top_level: description: >- The answer must return in these many results - range: string + range: integer - node: + query_node: description: >- - The node of the TRAPI query to replace. - range: string + The node of the (templated) TRAPI query to replace + range: NodeEnum notes: description: >- @@ -376,17 +406,24 @@ slots: range: string requires_trapi: - range: boolean description: >- Does this test require a TRAPI-compliant query to run? + range: boolean - inputs: - range: Input - multivalued: true + test_env: + description: >- + Deployment environment within which the associated TestSuite is run. + range: TestEnvEnum - outputs: - range: Output - multivalued: true + test_case_type: + description: >- + Type of TestCase. + range: TestCaseTypeEnum + + query_type: + description: >- + Type of TestCase query. + range: QueryTypeEnum preconditions: range: Precondition @@ -435,11 +472,21 @@ enums: Someone working on basic biology questions or drug discoveries where the study of the biological mechanism. + QueryTypeEnum: + description: >- + Query + permissible_values: + treats: + aliases: "treats(creative)" + description: + ExpectedOutputEnum: description: >- Expected output values for instances of Test Asset or Test Cases(?). (Note: does this Enum overlap with 'ExpectedResultsEnum' below?) permissible_values: + Top_Answer: + description: Acceptable: description: BadButForgivable: @@ -447,8 +494,10 @@ enums: NeverShow: description: 1_TopAnswer: + is_a: Top_Answer description: 4_NeverShow: + is_a: NeverShow description: ExpectedResultsEnum: @@ -462,12 +511,50 @@ enums: description: >- The query should not return the result in this test case + NodeEnum: + description: >- + Target node of a Subject-Predicate-Object driven query + permissible_values: + subject: + object: + + TestEnvEnum: + description: >- + Testing environments within which a TestSuite is + run by a TestRunner scheduled by the TestHarness. + permissible_values: + dev: + description: Development + ci: + description: Continuous Integration + test: + description: Test + prod: + description: Production + + TestCaseTypeEnum: + description: >- + Enumerated tags for types of test (generally applied to a TestCase). + permissible_values: + acceptance: + description: Acceptance test + quantitative: + description: Quantitative test + compliance: + description: Standards compliance test + kg_navigation: + description: Knowledge Graph navigation integration test + one_hop: + is_a: kg_navigation + description: One Hop navigation test + TestIssueEnum: permissible_values: causes not treats: description: TMKP: - description: Text Mining Knowledge Provider generated relationship? + description: >- + 'Text Mining Knowledge Provider' generated relationship? category too generic: description: contraindications: @@ -489,10 +576,3 @@ enums: permissible_values: increased: decreased: - - EnvironmentEnum: - permissible_values: - DEV: - CI: - TEST: - PROD: