diff --git a/404.html b/404.html index edda0838..9dd71062 100644 --- a/404.html +++ b/404.html @@ -11,7 +11,7 @@ - +
diff --git a/assets/js/60c4aef6.df20c7dd.js b/assets/js/60c4aef6.df20c7dd.js deleted file mode 100644 index 44bf2f7c..00000000 --- a/assets/js/60c4aef6.df20c7dd.js +++ /dev/null @@ -1 +0,0 @@ -"use strict";(self.webpackChunkbpd_reference=self.webpackChunkbpd_reference||[]).push([[6142],{3071:(s,e,a)=>{a.r(e),a.d(e,{assets:()=>c,contentTitle:()=>l,default:()=>d,frontMatter:()=>t,metadata:()=>r,toc:()=>m});var n=a(4848),i=a(8453);a(8597),a(5158);const t={sidebar_position:5,sidebar_label:"Observed & Test Statistic",title:"Observed & Test Statistic"},l=void 0,r={id:"statistical-inference/obs_and_test_stats",title:"Observed & Test Statistic",description:"Concept",source:"@site/docs/statistical-inference/obs_and_test_stats.md",sourceDirName:"statistical-inference",slug:"/statistical-inference/obs_and_test_stats",permalink:"/bpd-reference/docs/statistical-inference/obs_and_test_stats",draft:!1,unlisted:!1,editUrl:"https://github.com/dsc-courses/bpd-reference/tree/main/docs/statistical-inference/obs_and_test_stats.md",tags:[],version:"current",sidebarPosition:5,frontMatter:{sidebar_position:5,sidebar_label:"Observed & Test Statistic",title:"Observed & Test Statistic"},sidebar:"statsSidebar",previous:{title:"Null & Alternative Hypothesis",permalink:"/bpd-reference/docs/statistical-inference/null_and_alt"},next:{title:"P-value",permalink:"/bpd-reference/docs/statistical-inference/p_value"}},c={},m=[{value:"Concept",id:"concept",level:2},{value:"Common Test Statistics \ud83c\udf1f",id:"common-test-statistics-",level:2},{value:"1. Absolute Difference",id:"1-absolute-difference",level:3},{value:"2. Difference",id:"2-difference",level:3},{value:"3. Total Variation Distance (TVD)",id:"3-total-variation-distance-tvd",level:3}];function h(s){const e={a:"a",admonition:"admonition",annotation:"annotation",br:"br",code:"code",em:"em",h2:"h2",h3:"h3",li:"li",math:"math",mfrac:"mfrac",mi:"mi",mn:"mn",mo:"mo",mrow:"mrow",msub:"msub",mtext:"mtext",ol:"ol",p:"p",pre:"pre",semantics:"semantics",span:"span",strong:"strong",ul:"ul",...(0,i.R)(),...s.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(e.h2,{id:"concept",children:"Concept"}),"\n",(0,n.jsxs)(e.admonition,{title:"Terminology",type:"info",children:[(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Experiment"}),": A process whose outcome is random."]}),(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"Example: Flipping 100 coins."}),"\n"]}),(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Observed Statistic"}),": A statistic computed from the observed data."]}),(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"Example: The number of heads observed."}),"\n"]}),(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Test Statistic"}),": A statistic computed from a sample generated under the assumption that the null hypothesis is true."]}),(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"Think of the test statistic a number you write down each time you perform an experiment."}),"\n",(0,n.jsx)(e.li,{children:"The test statistic should be such that high observed values lean towards one hypothesis and low observed values lean towards the other."}),"\n"]})]}),"\n",(0,n.jsx)(e.h2,{id:"common-test-statistics-",children:"Common Test Statistics \ud83c\udf1f"}),"\n",(0,n.jsx)(e.h3,{id:"1-absolute-difference",children:"1. Absolute Difference"}),"\n",(0,n.jsx)(e.p,{children:"Absolute difference in group mean/median/number of times a certain event happens."}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsxs)(e.li,{children:["\u2705 Used for measuring how different two ",(0,n.jsx)(e.strong,{children:"numerical distributions"})," are, and when the alternative hypothesis is ",(0,n.jsx)(e.strong,{children:(0,n.jsx)(e.em,{children:"not equal to"})}),'. For example, "the coin is biased" or "the probability of tossing a head is 0.5".']}),"\n",(0,n.jsxs)(e.li,{children:["\ud83d\udcbb ",(0,n.jsx)(e.a,{href:"https://dsc-courses.github.io/bpd-reference/docs/statistical-inference/permutation_test#code-example-absolute-difference",children:"Example"})," of using absolute difference as the test statistic in a permutation test."]}),"\n"]}),"\n",(0,n.jsx)(e.h3,{id:"2-difference",children:"2. Difference"}),"\n",(0,n.jsx)(e.p,{children:"Difference in group mean/median/number of times a certain event happens."}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsxs)(e.li,{children:["\u2705 Used for measuring how different two ",(0,n.jsx)(e.strong,{children:"numerical distributions"})," are, and the alternative hypothesis is ",(0,n.jsx)(e.strong,{children:(0,n.jsx)(e.em,{children:"less than"})})," or ",(0,n.jsx)(e.strong,{children:(0,n.jsx)(e.em,{children:"greater than"})}),'. For example, "the coin is biased towards heads" or "the probability of tossing a head is greater then 0.5".']}),"\n",(0,n.jsxs)(e.li,{children:["\ud83d\udcbb ",(0,n.jsx)(e.a,{href:"https://dsc-courses.github.io/bpd-reference/docs/statistical-inference/hypothesis_test#code-example-1-difference",children:"Example"})," of using difference as the test statistic in a hypothesis test."]}),"\n"]}),"\n",(0,n.jsx)(e.h3,{id:"3-total-variation-distance-tvd",children:"3. Total Variation Distance (TVD)"}),"\n",(0,n.jsxs)(e.p,{children:["A test statistic that quantifies how different ",(0,n.jsx)(e.strong,{children:"two categorical distributions"})," are by calculating the sum of the absolute differences of their proportions, all divided by 2."]}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"\u274c\ufe0f The TVD is not used for permutation tests."}),"\n",(0,n.jsxs)(e.li,{children:['\u2705 Used for assessing whether an "observed sample" was drawn randomly from a known ',(0,n.jsx)(e.strong,{children:"categorical distribution"}),"."]}),"\n",(0,n.jsxs)(e.li,{children:["\ud83d\udcbb ",(0,n.jsx)(e.a,{href:"https://dsc-courses.github.io/bpd-reference/docs/statistical-inference/hypothesis_test/#code-example-2-tvd",children:"Example"})," of using TVD as the test statistic in a hypothesis test."]}),"\n"]}),"\n",(0,n.jsx)(e.pre,{children:(0,n.jsx)(e.code,{className:"language-python",children:"#code implementation\ndef tvd(dist1, dist2):\n '''Computes the TVD between two categorical distributions, \n assuming the categories appear in the same order.'''\n return np.abs(dist1 - dist2).sum() / 2\n"})}),"\n",(0,n.jsxs)(e.admonition,{type:"note",children:[(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"3 Ways of Manually Computing TVD:"})," \ud83e\uddee"]}),(0,n.jsx)(e.p,{children:"to do: use an example"}),(0,n.jsx)(e.p,{children:(0,n.jsxs)(e.em,{children:["Assume ",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"P"}),(0,n.jsx)(e.mi,{children:"i"})]})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"P_i"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.8333em",verticalAlign:"-0.15em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]})]})})]})," is one distribution and ",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"Q"}),(0,n.jsx)(e.mi,{children:"i"})]})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"Q_i"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.8778em",verticalAlign:"-0.1944em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]})]})})]})," is the other, and the categories appear in the same order"]})}),(0,n.jsxs)(e.ol,{children:["\n",(0,n.jsxs)(e.li,{children:["\n",(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Follow the definition:"})," Calculate the sum of the absolute differences of the two distributions' proportions, all divided by 2. ",(0,n.jsx)(e.br,{}),"\n",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mtext,{children:"TVD"}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsxs)(e.mfrac,{children:[(0,n.jsx)(e.mn,{children:"1"}),(0,n.jsx)(e.mn,{children:"2"})]}),(0,n.jsx)(e.mo,{children:"\u2211"}),(0,n.jsx)(e.mo,{children:"\u2223"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"P"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"Q"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{children:"\u2223"})]}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\text{TVD}=\\frac{1}{2}\\sum \\mid P_i - Q_i\\mid"})]})})}),(0,n.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.6833em"}}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"TVD"})}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1.1901em",verticalAlign:"-0.345em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mopen nulldelimiter"}),(0,n.jsx)(e.span,{className:"mfrac",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsxs)(e.span,{className:"vlist",style:{height:"0.8451em"},children:[(0,n.jsxs)(e.span,{style:{top:"-2.655em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:"2"})})})]}),(0,n.jsxs)(e.span,{style:{top:"-3.23em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"frac-line",style:{borderBottomWidth:"0.04em"}})]}),(0,n.jsxs)(e.span,{style:{top:"-3.394em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:(0,n.jsx)(e.span,{className:"mord mtight",children:"1"})})})]})]}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.345em"},children:(0,n.jsx)(e.span,{})})})]})}),(0,n.jsx)(e.span,{className:"mclose nulldelimiter"})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mop op-symbol small-op",style:{position:"relative",top:"0em"},children:"\u2211"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"\u2223"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.8333em",verticalAlign:"-0.15em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"\u2223"})]})]})]})]}),"\n"]}),"\n",(0,n.jsxs)(e.li,{children:["\n",(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Sum of positive differences:"})," Add only the values where one distribution (e.g. P_i) is greater than the other (e.g. Q_i) . This essentially sums the excessive probabilities in one distribution over the other. ",(0,n.jsx)(e.br,{}),"\n",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mtext,{children:"TVD"}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsx)(e.mo,{children:"\u2211"}),(0,n.jsx)(e.mi,{children:"m"}),(0,n.jsx)(e.mi,{children:"a"}),(0,n.jsx)(e.mi,{children:"x"}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"P"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"Q"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{separator:"true",children:","}),(0,n.jsx)(e.mn,{children:"0"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"})]}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\text{TVD}=\\sum max(P_i-Q_i,0)"})]})})}),(0,n.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.6833em"}}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"TVD"})}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsx)(e.span,{className:"mop op-symbol small-op",style:{position:"relative",top:"0em"},children:"\u2211"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"ma"}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mpunct",children:","}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0"}),(0,n.jsx)(e.span,{className:"mclose",children:")"})]})]})]})]}),"\n"]}),"\n"]}),(0,n.jsx)(e.p,{children:"to do: replace formula with example DataFrame visualization"}),(0,n.jsxs)(e.ol,{start:"3",children:["\n",(0,n.jsxs)(e.li,{children:[(0,n.jsx)(e.strong,{children:"Sum of negative differences:"})," Add only the absolute values where the selected distribution (e.g. P_i) is less than the other (e.g. Q_i). This essentially sums the deficient probabilities in one distribution relative to the other. ",(0,n.jsx)(e.br,{}),"\n",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mtext,{children:"TVD"}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsx)(e.mo,{children:"\u2211"}),(0,n.jsx)(e.mi,{children:"a"}),(0,n.jsx)(e.mi,{children:"b"}),(0,n.jsx)(e.mi,{children:"s"}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsx)(e.mi,{children:"m"}),(0,n.jsx)(e.mi,{children:"i"}),(0,n.jsx)(e.mi,{children:"n"}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"P"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"Q"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{separator:"true",children:","}),(0,n.jsx)(e.mn,{children:"0"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"}),(0,n.jsx)(e.mtext,{children:"\xa0or\xa0TVD"}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsx)(e.mo,{children:"\u2211"}),(0,n.jsx)(e.mi,{children:"m"}),(0,n.jsx)(e.mi,{children:"a"}),(0,n.jsx)(e.mi,{children:"x"}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"Q"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"P"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{separator:"true",children:","}),(0,n.jsx)(e.mn,{children:"0"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"})]}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\text{TVD}=\\sum abs(min(P_i-Q_i,0)) \\ \\text{or} \\ \\text{TVD}=\\sum max(Q_i-P_i,0)"})]})})}),(0,n.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.6833em"}}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"TVD"})}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsx)(e.span,{className:"mop op-symbol small-op",style:{position:"relative",top:"0em"},children:"\u2211"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"ab"}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"s"}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"min"}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mpunct",children:","}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0"}),(0,n.jsx)(e.span,{className:"mclose",children:"))"}),(0,n.jsx)(e.span,{className:"mspace",children:"\xa0"}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"or"})}),(0,n.jsx)(e.span,{className:"mspace",children:"\xa0"}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"TVD"})}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsx)(e.span,{className:"mop op-symbol small-op",style:{position:"relative",top:"0em"},children:"\u2211"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"ma"}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mpunct",children:","}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0"}),(0,n.jsx)(e.span,{className:"mclose",children:")"})]})]})]})]}),"\n"]}),(0,n.jsx)(e.p,{children:"to do: replace formula with example DataFrame visualization"})]})]})}function d(s={}){const{wrapper:e}={...(0,i.R)(),...s.components};return e?(0,n.jsx)(e,{...s,children:(0,n.jsx)(h,{...s})}):h(s)}},8597:(s,e,a)=>{a.d(e,{A:()=>i});a(6540);var n=a(4848);const i=s=>{let{data:e}=s,a=[],i=[],t="",l=[];if(e){const s=JSON.parse(e);a=s.columns||[],i=s.data||[],t=s.indexName||"Index",l=s.index||[]}return(0,n.jsx)("div",{className:"dataframe-container",children:(0,n.jsxs)("table",{className:"dataframe-table",children:[(0,n.jsx)("thead",{children:(0,n.jsxs)("tr",{children:[(0,n.jsx)("th",{children:t}),a.map(((s,e)=>(0,n.jsx)("th",{children:s},e)))]})}),(0,n.jsx)("tbody",{children:i.map(((s,e)=>(0,n.jsxs)("tr",{children:[(0,n.jsx)("td",{className:"index-value",children:l[e]}),s.map(((s,e)=>{return(0,n.jsx)("td",{children:(a=s,"boolean"==typeof a?a.toString().charAt(0).toUpperCase()+a.toString().slice(1):a)},e);var a}))]},e)))})]})})}},5158:(s,e,a)=>{a.d(e,{A:()=>t});var n=a(6540),i=a(4848);const t=function(s){let{src:e,sourceLink:a,aspectRatio:t=16/9}=s;const[l,r]=(0,n.useState)(0),[c,m]=(0,n.useState)(0),h=(0,n.useRef)(null);return(0,n.useEffect)((()=>{const s=()=>{if(h.current){const s=h.current.offsetWidth;m(s/t)}};return s(),window.addEventListener("resize",s),()=>window.removeEventListener("resize",s)}),[t]),(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)("div",{ref:h,style:{width:"100%",position:"relative",maxWidth:"100%",backgroundColor:"white"},children:(0,i.jsx)("iframe",{src:e,style:{width:"100%",height:`${c}px`,border:"0",allowFullScreen:!0}},l)}),(0,i.jsx)("div",{style:{display:"flex",justifyContent:"center",margin:"20px 0"},children:(0,i.jsx)("a",{href:a,target:"_blank",rel:"noopener noreferrer",children:"(Source)"})}),(0,i.jsx)("div",{style:{display:"flex",justifyContent:"center",margin:"10px 0"},children:(0,i.jsx)("button",{onClick:()=>{r((s=>s+1))},style:{backgroundColor:"#4CAF50",color:"white",padding:"10px 20px",border:"none",borderRadius:"5px",cursor:"pointer",outline:"none",fontSize:"16px",fontWeight:"bold",boxShadow:"0 4px 8px rgba(0,0,0,0.1)",transition:"background-color 0.3s"},onMouseOver:s=>s.currentTarget.style.backgroundColor="#45a049",onMouseOut:s=>s.currentTarget.style.backgroundColor="#4CAF50",children:"Restart Diagram"})})]})}},8453:(s,e,a)=>{a.d(e,{R:()=>l,x:()=>r});var n=a(6540);const i={},t=n.createContext(i);function l(s){const e=n.useContext(t);return n.useMemo((function(){return"function"==typeof s?s(e):{...e,...s}}),[e,s])}function r(s){let e;return e=s.disableParentContext?"function"==typeof s.components?s.components(i):s.components||i:l(s.components),n.createElement(t.Provider,{value:e},s.children)}}}]); \ No newline at end of file diff --git a/assets/js/60c4aef6.f5ab945a.js b/assets/js/60c4aef6.f5ab945a.js new file mode 100644 index 00000000..b40c15e6 --- /dev/null +++ b/assets/js/60c4aef6.f5ab945a.js @@ -0,0 +1 @@ +"use strict";(self.webpackChunkbpd_reference=self.webpackChunkbpd_reference||[]).push([[6142],{3071:(s,e,a)=>{a.r(e),a.d(e,{assets:()=>m,contentTitle:()=>r,default:()=>x,frontMatter:()=>t,metadata:()=>c,toc:()=>h});var n=a(4848),l=a(8453),i=a(8597);a(5158);const t={sidebar_position:5,sidebar_label:"Observed & Test Statistic",title:"Observed & Test Statistic"},r=void 0,c={id:"statistical-inference/obs_and_test_stats",title:"Observed & Test Statistic",description:"Concept",source:"@site/docs/statistical-inference/obs_and_test_stats.md",sourceDirName:"statistical-inference",slug:"/statistical-inference/obs_and_test_stats",permalink:"/bpd-reference/docs/statistical-inference/obs_and_test_stats",draft:!1,unlisted:!1,editUrl:"https://github.com/dsc-courses/bpd-reference/tree/main/docs/statistical-inference/obs_and_test_stats.md",tags:[],version:"current",sidebarPosition:5,frontMatter:{sidebar_position:5,sidebar_label:"Observed & Test Statistic",title:"Observed & Test Statistic"},sidebar:"statsSidebar",previous:{title:"Null & Alternative Hypothesis",permalink:"/bpd-reference/docs/statistical-inference/null_and_alt"},next:{title:"P-value",permalink:"/bpd-reference/docs/statistical-inference/p_value"}},m={},h=[{value:"Concept",id:"concept",level:2},{value:"Common Test Statistics \ud83c\udf1f",id:"common-test-statistics-",level:2},{value:"1. Absolute Difference",id:"1-absolute-difference",level:3},{value:"2. Difference",id:"2-difference",level:3},{value:"3. Total Variation Distance (TVD)",id:"3-total-variation-distance-tvd",level:3}];function d(s){const e={a:"a",admonition:"admonition",annotation:"annotation",br:"br",code:"code",em:"em",h2:"h2",h3:"h3",li:"li",math:"math",mfrac:"mfrac",mi:"mi",mn:"mn",mo:"mo",mrow:"mrow",mstyle:"mstyle",msub:"msub",mtable:"mtable",mtd:"mtd",mtext:"mtext",mtr:"mtr",ol:"ol",p:"p",pre:"pre",semantics:"semantics",span:"span",strong:"strong",ul:"ul",...(0,l.R)(),...s.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(e.h2,{id:"concept",children:"Concept"}),"\n",(0,n.jsxs)(e.admonition,{title:"Terminology",type:"info",children:[(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Experiment"}),": A process whose outcome is random."]}),(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"Example: Flipping 100 coins."}),"\n"]}),(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Observed Statistic"}),": A statistic computed from the observed data."]}),(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"Example: The number of heads observed."}),"\n"]}),(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Test Statistic"}),": A statistic computed from a sample generated under the assumption that the null hypothesis is true."]}),(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"Think of the test statistic a number you write down each time you perform an experiment."}),"\n",(0,n.jsx)(e.li,{children:"The test statistic should be such that high observed values lean towards one hypothesis and low observed values lean towards the other."}),"\n"]})]}),"\n",(0,n.jsx)(e.h2,{id:"common-test-statistics-",children:"Common Test Statistics \ud83c\udf1f"}),"\n",(0,n.jsx)(e.h3,{id:"1-absolute-difference",children:"1. Absolute Difference"}),"\n",(0,n.jsx)(e.p,{children:"Absolute difference in group mean/median/number of times a certain event happens."}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsxs)(e.li,{children:["\u2705 Used for measuring how different two ",(0,n.jsx)(e.strong,{children:"numerical distributions"})," are, and when the alternative hypothesis is ",(0,n.jsx)(e.strong,{children:(0,n.jsx)(e.em,{children:"not equal to"})}),'. For example, "the coin is biased" or "the probability of tossing a head is 0.5".']}),"\n",(0,n.jsxs)(e.li,{children:["\ud83d\udcbb ",(0,n.jsx)(e.a,{href:"https://dsc-courses.github.io/bpd-reference/docs/statistical-inference/permutation_test#code-example-absolute-difference",children:"Example"})," of using absolute difference as the test statistic in a permutation test."]}),"\n"]}),"\n",(0,n.jsx)(e.h3,{id:"2-difference",children:"2. Difference"}),"\n",(0,n.jsx)(e.p,{children:"Difference in group mean/median/number of times a certain event happens."}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsxs)(e.li,{children:["\u2705 Used for measuring how different two ",(0,n.jsx)(e.strong,{children:"numerical distributions"})," are, and the alternative hypothesis is ",(0,n.jsx)(e.strong,{children:(0,n.jsx)(e.em,{children:"less than"})})," or ",(0,n.jsx)(e.strong,{children:(0,n.jsx)(e.em,{children:"greater than"})}),'. For example, "the coin is biased towards heads" or "the probability of tossing a head is greater then 0.5".']}),"\n",(0,n.jsxs)(e.li,{children:["\ud83d\udcbb ",(0,n.jsx)(e.a,{href:"https://dsc-courses.github.io/bpd-reference/docs/statistical-inference/hypothesis_test#code-example-1-difference",children:"Example"})," of using difference as the test statistic in a hypothesis test."]}),"\n"]}),"\n",(0,n.jsx)(e.h3,{id:"3-total-variation-distance-tvd",children:"3. Total Variation Distance (TVD)"}),"\n",(0,n.jsxs)(e.p,{children:["A test statistic that quantifies how different ",(0,n.jsx)(e.strong,{children:"two categorical distributions"})," are by calculating the sum of the absolute differences of their proportions, all divided by 2."]}),"\n",(0,n.jsxs)(e.ul,{children:["\n",(0,n.jsx)(e.li,{children:"\u274c\ufe0f The TVD is not used for permutation tests."}),"\n",(0,n.jsxs)(e.li,{children:['\u2705 Used for assessing whether an "observed sample" was drawn randomly from a known ',(0,n.jsx)(e.strong,{children:"categorical distribution"}),"."]}),"\n",(0,n.jsxs)(e.li,{children:["\ud83d\udcbb ",(0,n.jsx)(e.a,{href:"https://dsc-courses.github.io/bpd-reference/docs/statistical-inference/hypothesis_test/#code-example-2-tvd",children:"Example"})," of using TVD as the test statistic in a hypothesis test."]}),"\n"]}),"\n",(0,n.jsx)(e.pre,{children:(0,n.jsx)(e.code,{className:"language-python",children:"#code implementation\ndef tvd(dist1, dist2):\n '''Computes the TVD between two categorical distributions, \n assuming the categories appear in the same order.'''\n return np.abs(dist1 - dist2).sum() / 2\n"})}),"\n",(0,n.jsxs)(e.admonition,{type:"note",children:[(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"3 Ways of Manually Computing TVD:"})," \ud83e\uddee"]}),(0,n.jsx)(e.p,{children:"Let's use an example to show how TVD can be computed in three ways."}),(0,n.jsxs)(e.p,{children:["In the ",(0,n.jsx)(e.code,{children:"full_pets"})," DataFrame, we found that the color distribution of dogs seems different from that of all pets."]}),(0,n.jsx)(i.A,{data:'{"columns":["Pets_Dist","Dogs_Dist"],"index":["black","golden","white"],"data":[[0.53,0.44],[0.21,0.12],[0.26,0.44]]}'}),(0,n.jsx)(e.p,{children:(0,n.jsxs)(e.em,{children:["Assume ",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsx)(e.mi,{children:"P"})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"P"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.6833em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"})]})})]})," is one distribution and ",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsx)(e.mrow,{children:(0,n.jsx)(e.mi,{children:"Q"})}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"Q"})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.8778em",verticalAlign:"-0.1944em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"})]})})]})," is the other, and the categories appear in the same order."]})}),(0,n.jsxs)(e.ol,{children:["\n",(0,n.jsxs)(e.li,{children:["\n",(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Follow the definition:"})," Calculate the sum of the absolute differences of the proportions of the two distributions (P and Q), all divided by 2."]}),"\n",(0,n.jsx)(e.span,{className:"katex-display",children:(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",display:"block",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsxs)(e.mtable,{rowspacing:"0.25em",columnalign:"right left",columnspacing:"0em",children:[(0,n.jsxs)(e.mtr,{children:[(0,n.jsx)(e.mtd,{children:(0,n.jsx)(e.mstyle,{scriptlevel:"0",displaystyle:"true",children:(0,n.jsx)(e.mtext,{children:"TVD"})})}),(0,n.jsx)(e.mtd,{children:(0,n.jsx)(e.mstyle,{scriptlevel:"0",displaystyle:"true",children:(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mrow,{}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsxs)(e.mfrac,{children:[(0,n.jsx)(e.mn,{children:"1"}),(0,n.jsx)(e.mn,{children:"2"})]}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"\u2223"}),(0,n.jsx)(e.mn,{children:"0.53"}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsx)(e.mn,{children:"0.44"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"\u2223"}),(0,n.jsx)(e.mo,{children:"+"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"\u2223"}),(0,n.jsx)(e.mn,{children:"0.21"}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsx)(e.mn,{children:"0.12"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"\u2223"}),(0,n.jsx)(e.mo,{children:"+"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"\u2223"}),(0,n.jsx)(e.mn,{children:"0.26"}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsx)(e.mn,{children:"0.44"}),(0,n.jsx)(e.mi,{mathvariant:"normal",children:"\u2223"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"})]})})})]}),(0,n.jsxs)(e.mtr,{children:[(0,n.jsx)(e.mtd,{children:(0,n.jsx)(e.mstyle,{scriptlevel:"0",displaystyle:"true",children:(0,n.jsx)(e.mrow,{})})}),(0,n.jsx)(e.mtd,{children:(0,n.jsx)(e.mstyle,{scriptlevel:"0",displaystyle:"true",children:(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mrow,{}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsxs)(e.mfrac,{children:[(0,n.jsx)(e.mn,{children:"1"}),(0,n.jsx)(e.mn,{children:"2"})]}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsx)(e.mn,{children:"0.09"}),(0,n.jsx)(e.mo,{children:"+"}),(0,n.jsx)(e.mn,{children:"0.09"}),(0,n.jsx)(e.mo,{children:"+"}),(0,n.jsx)(e.mn,{children:"0.18"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"})]})})})]}),(0,n.jsxs)(e.mtr,{children:[(0,n.jsx)(e.mtd,{children:(0,n.jsx)(e.mstyle,{scriptlevel:"0",displaystyle:"true",children:(0,n.jsx)(e.mrow,{})})}),(0,n.jsx)(e.mtd,{children:(0,n.jsx)(e.mstyle,{scriptlevel:"0",displaystyle:"true",children:(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mrow,{}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsx)(e.mn,{children:"0.18"})]})})})]})]}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\begin{align*} \n\\text{TVD}\n&= \\frac{1}{2}(|0.53-0.44|+|0.21-0.12|+|0.26-0.44|) \\\\\n&= \\frac{1}{2}(0.09+0.09+0.18) \\\\\n&= 0.18\n\\end{align*} "})]})})}),(0,n.jsx)(e.span,{className:"katex-html","aria-hidden":"true",children:(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"6.1149em",verticalAlign:"-2.8074em"}}),(0,n.jsx)(e.span,{className:"mord",children:(0,n.jsxs)(e.span,{className:"mtable",children:[(0,n.jsx)(e.span,{className:"col-align-r",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsxs)(e.span,{className:"vlist",style:{height:"3.3074em"},children:[(0,n.jsxs)(e.span,{style:{top:"-5.3074em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3.3214em"}}),(0,n.jsx)(e.span,{className:"mord",children:(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"TVD"})})})]}),(0,n.jsxs)(e.span,{style:{top:"-3em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3.3214em"}}),(0,n.jsx)(e.span,{className:"mord"})]}),(0,n.jsxs)(e.span,{style:{top:"-1.174em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3.3214em"}}),(0,n.jsx)(e.span,{className:"mord"})]})]}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"2.8074em"},children:(0,n.jsx)(e.span,{})})})]})}),(0,n.jsx)(e.span,{className:"col-align-l",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsxs)(e.span,{className:"vlist",style:{height:"3.3074em"},children:[(0,n.jsxs)(e.span,{style:{top:"-5.3074em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3.3214em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mopen nulldelimiter"}),(0,n.jsx)(e.span,{className:"mfrac",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsxs)(e.span,{className:"vlist",style:{height:"1.3214em"},children:[(0,n.jsxs)(e.span,{style:{top:"-2.314em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"mord",children:(0,n.jsx)(e.span,{className:"mord",children:"2"})})]}),(0,n.jsxs)(e.span,{style:{top:"-3.23em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"frac-line",style:{borderBottomWidth:"0.04em"}})]}),(0,n.jsxs)(e.span,{style:{top:"-3.677em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"mord",children:(0,n.jsx)(e.span,{className:"mord",children:"1"})})]})]}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.686em"},children:(0,n.jsx)(e.span,{})})})]})}),(0,n.jsx)(e.span,{className:"mclose nulldelimiter"})]}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsx)(e.span,{className:"mord",children:"\u22230.53"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0.44\u2223"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"+"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mord",children:"\u22230.21"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0.12\u2223"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"+"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mord",children:"\u22230.26"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0.44\u2223"}),(0,n.jsx)(e.span,{className:"mclose",children:")"})]})]}),(0,n.jsxs)(e.span,{style:{top:"-3em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3.3214em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mopen nulldelimiter"}),(0,n.jsx)(e.span,{className:"mfrac",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsxs)(e.span,{className:"vlist",style:{height:"1.3214em"},children:[(0,n.jsxs)(e.span,{style:{top:"-2.314em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"mord",children:(0,n.jsx)(e.span,{className:"mord",children:"2"})})]}),(0,n.jsxs)(e.span,{style:{top:"-3.23em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"frac-line",style:{borderBottomWidth:"0.04em"}})]}),(0,n.jsxs)(e.span,{style:{top:"-3.677em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3em"}}),(0,n.jsx)(e.span,{className:"mord",children:(0,n.jsx)(e.span,{className:"mord",children:"1"})})]})]}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.686em"},children:(0,n.jsx)(e.span,{})})})]})}),(0,n.jsx)(e.span,{className:"mclose nulldelimiter"})]}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsx)(e.span,{className:"mord",children:"0.09"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"+"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0.09"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"+"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0.18"}),(0,n.jsx)(e.span,{className:"mclose",children:")"})]})]}),(0,n.jsxs)(e.span,{style:{top:"-1.174em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"3.3214em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0.18"})]})]})]}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"2.8074em"},children:(0,n.jsx)(e.span,{})})})]})})]})})]})})]})}),"\n"]}),"\n",(0,n.jsxs)(e.li,{children:["\n",(0,n.jsxs)(e.p,{children:[(0,n.jsx)(e.strong,{children:"Sum of positive differences:"})," Add only the values where one distribution (e.g. P_i) is greater than the other (e.g. Q_i) . This essentially sums the excessive probabilities in one distribution over the other. ",(0,n.jsx)(e.br,{}),"\n",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mtext,{children:"TVD"}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsx)(e.mo,{children:"\u2211"}),(0,n.jsx)(e.mi,{children:"m"}),(0,n.jsx)(e.mi,{children:"a"}),(0,n.jsx)(e.mi,{children:"x"}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"P"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"Q"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{separator:"true",children:","}),(0,n.jsx)(e.mn,{children:"0"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"})]}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\text{TVD}=\\sum max(P_i-Q_i,0)"})]})})}),(0,n.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.6833em"}}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"TVD"})}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsx)(e.span,{className:"mop op-symbol small-op",style:{position:"relative",top:"0em"},children:"\u2211"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"ma"}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mpunct",children:","}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0"}),(0,n.jsx)(e.span,{className:"mclose",children:")"})]})]})]})]}),"\n"]}),"\n"]}),(0,n.jsx)(e.p,{children:"to do: replace formula with example DataFrame visualization"}),(0,n.jsxs)(e.ol,{start:"3",children:["\n",(0,n.jsxs)(e.li,{children:[(0,n.jsx)(e.strong,{children:"Sum of negative differences:"})," Add only the absolute values where the selected distribution (e.g. P_i) is less than the other (e.g. Q_i). This essentially sums the deficient probabilities in one distribution relative to the other. ",(0,n.jsx)(e.br,{}),"\n",(0,n.jsxs)(e.span,{className:"katex",children:[(0,n.jsx)(e.span,{className:"katex-mathml",children:(0,n.jsx)(e.math,{xmlns:"http://www.w3.org/1998/Math/MathML",children:(0,n.jsxs)(e.semantics,{children:[(0,n.jsxs)(e.mrow,{children:[(0,n.jsx)(e.mtext,{children:"TVD"}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsx)(e.mo,{children:"\u2211"}),(0,n.jsx)(e.mi,{children:"a"}),(0,n.jsx)(e.mi,{children:"b"}),(0,n.jsx)(e.mi,{children:"s"}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsx)(e.mi,{children:"m"}),(0,n.jsx)(e.mi,{children:"i"}),(0,n.jsx)(e.mi,{children:"n"}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"P"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"Q"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{separator:"true",children:","}),(0,n.jsx)(e.mn,{children:"0"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"}),(0,n.jsx)(e.mtext,{children:"\xa0or\xa0TVD"}),(0,n.jsx)(e.mo,{children:"="}),(0,n.jsx)(e.mo,{children:"\u2211"}),(0,n.jsx)(e.mi,{children:"m"}),(0,n.jsx)(e.mi,{children:"a"}),(0,n.jsx)(e.mi,{children:"x"}),(0,n.jsx)(e.mo,{stretchy:"false",children:"("}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"Q"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{children:"\u2212"}),(0,n.jsxs)(e.msub,{children:[(0,n.jsx)(e.mi,{children:"P"}),(0,n.jsx)(e.mi,{children:"i"})]}),(0,n.jsx)(e.mo,{separator:"true",children:","}),(0,n.jsx)(e.mn,{children:"0"}),(0,n.jsx)(e.mo,{stretchy:"false",children:")"})]}),(0,n.jsx)(e.annotation,{encoding:"application/x-tex",children:"\\text{TVD}=\\sum abs(min(P_i-Q_i,0)) \\ \\text{or} \\ \\text{TVD}=\\sum max(Q_i-P_i,0)"})]})})}),(0,n.jsxs)(e.span,{className:"katex-html","aria-hidden":"true",children:[(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"0.6833em"}}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"TVD"})}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsx)(e.span,{className:"mop op-symbol small-op",style:{position:"relative",top:"0em"},children:"\u2211"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"ab"}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"s"}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"min"}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mpunct",children:","}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0"}),(0,n.jsx)(e.span,{className:"mclose",children:"))"}),(0,n.jsx)(e.span,{className:"mspace",children:"\xa0"}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"or"})}),(0,n.jsx)(e.span,{className:"mspace",children:"\xa0"}),(0,n.jsx)(e.span,{className:"mord text",children:(0,n.jsx)(e.span,{className:"mord",children:"TVD"})}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}}),(0,n.jsx)(e.span,{className:"mrel",children:"="}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2778em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsx)(e.span,{className:"mop op-symbol small-op",style:{position:"relative",top:"0em"},children:"\u2211"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"ma"}),(0,n.jsx)(e.span,{className:"mord mathnormal",children:"x"}),(0,n.jsx)(e.span,{className:"mopen",children:"("}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",children:"Q"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}}),(0,n.jsx)(e.span,{className:"mbin",children:"\u2212"}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.2222em"}})]}),(0,n.jsxs)(e.span,{className:"base",children:[(0,n.jsx)(e.span,{className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,n.jsxs)(e.span,{className:"mord",children:[(0,n.jsx)(e.span,{className:"mord mathnormal",style:{marginRight:"0.13889em"},children:"P"}),(0,n.jsx)(e.span,{className:"msupsub",children:(0,n.jsxs)(e.span,{className:"vlist-t vlist-t2",children:[(0,n.jsxs)(e.span,{className:"vlist-r",children:[(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.3117em"},children:(0,n.jsxs)(e.span,{style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"},children:[(0,n.jsx)(e.span,{className:"pstrut",style:{height:"2.7em"}}),(0,n.jsx)(e.span,{className:"sizing reset-size6 size3 mtight",children:(0,n.jsx)(e.span,{className:"mord mathnormal mtight",children:"i"})})]})}),(0,n.jsx)(e.span,{className:"vlist-s",children:"\u200b"})]}),(0,n.jsx)(e.span,{className:"vlist-r",children:(0,n.jsx)(e.span,{className:"vlist",style:{height:"0.15em"},children:(0,n.jsx)(e.span,{})})})]})})]}),(0,n.jsx)(e.span,{className:"mpunct",children:","}),(0,n.jsx)(e.span,{className:"mspace",style:{marginRight:"0.1667em"}}),(0,n.jsx)(e.span,{className:"mord",children:"0"}),(0,n.jsx)(e.span,{className:"mclose",children:")"})]})]})]})]}),"\n"]}),(0,n.jsx)(e.p,{children:"to do: replace formula with example DataFrame visualization"})]})]})}function x(s={}){const{wrapper:e}={...(0,l.R)(),...s.components};return e?(0,n.jsx)(e,{...s,children:(0,n.jsx)(d,{...s})}):d(s)}},8597:(s,e,a)=>{a.d(e,{A:()=>l});a(6540);var n=a(4848);const l=s=>{let{data:e}=s,a=[],l=[],i="",t=[];if(e){const s=JSON.parse(e);a=s.columns||[],l=s.data||[],i=s.indexName||"Index",t=s.index||[]}return(0,n.jsx)("div",{className:"dataframe-container",children:(0,n.jsxs)("table",{className:"dataframe-table",children:[(0,n.jsx)("thead",{children:(0,n.jsxs)("tr",{children:[(0,n.jsx)("th",{children:i}),a.map(((s,e)=>(0,n.jsx)("th",{children:s},e)))]})}),(0,n.jsx)("tbody",{children:l.map(((s,e)=>(0,n.jsxs)("tr",{children:[(0,n.jsx)("td",{className:"index-value",children:t[e]}),s.map(((s,e)=>{return(0,n.jsx)("td",{children:(a=s,"boolean"==typeof a?a.toString().charAt(0).toUpperCase()+a.toString().slice(1):a)},e);var a}))]},e)))})]})})}},5158:(s,e,a)=>{a.d(e,{A:()=>i});var n=a(6540),l=a(4848);const i=function(s){let{src:e,sourceLink:a,aspectRatio:i=16/9}=s;const[t,r]=(0,n.useState)(0),[c,m]=(0,n.useState)(0),h=(0,n.useRef)(null);return(0,n.useEffect)((()=>{const s=()=>{if(h.current){const s=h.current.offsetWidth;m(s/i)}};return s(),window.addEventListener("resize",s),()=>window.removeEventListener("resize",s)}),[i]),(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)("div",{ref:h,style:{width:"100%",position:"relative",maxWidth:"100%",backgroundColor:"white"},children:(0,l.jsx)("iframe",{src:e,style:{width:"100%",height:`${c}px`,border:"0",allowFullScreen:!0}},t)}),(0,l.jsx)("div",{style:{display:"flex",justifyContent:"center",margin:"20px 0"},children:(0,l.jsx)("a",{href:a,target:"_blank",rel:"noopener noreferrer",children:"(Source)"})}),(0,l.jsx)("div",{style:{display:"flex",justifyContent:"center",margin:"10px 0"},children:(0,l.jsx)("button",{onClick:()=>{r((s=>s+1))},style:{backgroundColor:"#4CAF50",color:"white",padding:"10px 20px",border:"none",borderRadius:"5px",cursor:"pointer",outline:"none",fontSize:"16px",fontWeight:"bold",boxShadow:"0 4px 8px rgba(0,0,0,0.1)",transition:"background-color 0.3s"},onMouseOver:s=>s.currentTarget.style.backgroundColor="#45a049",onMouseOut:s=>s.currentTarget.style.backgroundColor="#4CAF50",children:"Restart Diagram"})})]})}},8453:(s,e,a)=>{a.d(e,{R:()=>t,x:()=>r});var n=a(6540);const l={},i=n.createContext(l);function t(s){const e=n.useContext(i);return n.useMemo((function(){return"function"==typeof s?s(e):{...e,...s}}),[e,s])}function r(s){let e;return e=s.disableParentContext?"function"==typeof s.components?s.components(l):s.components||l:t(s.components),n.createElement(i.Provider,{value:e},s.children)}}}]); \ No newline at end of file diff --git a/assets/js/runtime~main.cd6e20cf.js b/assets/js/runtime~main.142639b8.js similarity index 98% rename from assets/js/runtime~main.cd6e20cf.js rename to assets/js/runtime~main.142639b8.js index b949b20d..d8ace926 100644 --- a/assets/js/runtime~main.cd6e20cf.js +++ b/assets/js/runtime~main.142639b8.js @@ -1 +1 @@ -(()=>{"use strict";var e,a,f,c,d,b={},r={};function t(e){var a=r[e];if(void 0!==a)return a.exports;var f=r[e]={id:e,loaded:!1,exports:{}};return b[e].call(f.exports,f,f.exports,t),f.loaded=!0,f.exports}t.m=b,t.c=r,e=[],t.O=(a,f,c,d)=>{if(!f){var b=1/0;for(i=0;i#code implementation
def tvd(dist1, dist2):
'''Computes the TVD between two categorical distributions,
assuming the categories appear in the same order.'''
return np.abs(dist1 - dist2).sum() / 2
3 Ways of Manually Computing TVD: 🧮
to do: use an example
Assume is one distribution and is the other, and the categories appear in the same order
3 Ways of Manually Computing TVD: 🧮
Let's use an example to show how TVD can be computed in three ways.
In the full_pets
DataFrame, we found that the color distribution of dogs seems different from that of all pets.
Index | Pets_Dist | Dogs_Dist |
---|---|---|
black | 0.53 | 0.44 |
golden | 0.21 | 0.12 |
white | 0.26 | 0.44 |
Assume is one distribution and is the other, and the categories appear in the same order.
Follow the definition: Calculate the sum of the absolute differences of the two distributions' proportions, all divided by 2.
-
Follow the definition: Calculate the sum of the absolute differences of the proportions of the two distributions (P and Q), all divided by 2.
+Sum of positive differences: Add only the values where one distribution (e.g. P_i) is greater than the other (e.g. Q_i) . This essentially sums the excessive probabilities in one distribution over the other.
diff --git a/docs/statistical-inference/p_value/index.html b/docs/statistical-inference/p_value/index.html
index d06a3ae6..94daead3 100644
--- a/docs/statistical-inference/p_value/index.html
+++ b/docs/statistical-inference/p_value/index.html
@@ -11,7 +11,7 @@
-
+