%!PS-Adobe-1.0 %%Creator: gnoe.cwi.nl:paulv (P. Vitanyi,M339,4124,020-6252688) %%Title: stdin (ditroff) %%CreationDate: Thu Dec 4 17:27:03 1997 %%Pages: (atend) %%DocumentFonts: (atend) %%EndComments % Start of psdit.pro -- prolog for ditroff translator % Copyright (c) 1985,1987 Adobe Systems Incorporated. All Rights Reserved. % GOVERNMENT END USERS: See Notice file in TranScript library directory % -- probably /usr/lib/ps/Notice % RCS: $Header: psdit.pro,v 2.2 87/11/17 16:40:42 byron Rel $ /$DITroff 140 dict def $DITroff begin /fontnum 1 def /fontsize 10 def /fontheight 10 def /fontslant 0 def /xi {0 72 11.4 mul translate 72 resolution div dup neg scale F } def /m1 matrix def /m2 matrix def /m3 matrix def /oldmat matrix def /tan{dup sin exch cos div}bind def /point{resolution 72 div mul}bind def /dround {transform round exch round exch itransform}bind def /xT{/devname exch def}def /xr{/mh exch def /my exch def /resolution exch def}def /xp{}def /xs{docsave restore end}def /xt{}def /xf{/fontname exch def /slotno exch def fontnames slotno get fontname eq not {fonts slotno fontname findfont put fontnames slotno fontname put}if}def /xH{/fontheight exch def F}bind def /xS{/fontslant exch def F}bind def /s{/fontsize exch def /fontheight fontsize def F}bind def /f{/fontnum exch def F}bind def /F{fontheight 0 le {/fontheight fontsize def}if fonts fontnum get fontsize point 0 0 fontheight point neg 0 0 m1 astore fontslant 0 ne{1 0 fontslant tan 1 0 0 m2 astore m3 concatmatrix}if makefont setfont .04 fontsize point mul 0 dround pop abs setlinewidth}bind def /X{exch currentpoint exch pop moveto show}bind def /N{3 1 roll moveto show}bind def /Y{exch currentpoint pop exch moveto show}bind def /S /show load def /ditpush{}def/ditpop{}def /MX{currentpoint exch pop moveto}bind def /MY{currentpoint pop exch moveto}bind def /MXY /moveto load def /cb{pop}def % action on unknown char -- nothing for now /n{}def/w{}def /bp{/psv save def 0 0 moveto}def /ep{showpage psv restore}def /abspoint{currentpoint exch pop add exch currentpoint pop add exch}def /dstroke{currentpoint stroke moveto}bind def /Dl{2 copy gsave rlineto stroke grestore rmoveto}bind def /arcellipse{oldmat currentmatrix pop currentpoint translate 1 diamv diamh div scale /rad diamh 2 div def rad 0 rad -180 180 arc oldmat setmatrix}def /Dc{gsave dup /diamv exch def /diamh exch def arcellipse dstroke grestore diamh 0 rmoveto}def /De{gsave /diamv exch def /diamh exch def arcellipse dstroke grestore diamh 0 rmoveto}def /Da{currentpoint /by exch def /bx exch def /fy exch def /fx exch def /cy exch def /cx exch def /rad cx cx mul cy cy mul add sqrt def /ang1 cy neg cx neg atan def /ang2 fy fx atan def cx bx add cy by add 2 copy rad ang1 ang2 arcn stroke exch fx add exch fy add moveto}def /Barray 200 array def % 200 values in a wiggle /D~{mark}def /D~~{counttomark Barray exch 0 exch getinterval astore /Bcontrol exch def pop /Blen Bcontrol length def Blen 4 ge Blen 2 mod 0 eq and {Bcontrol 0 get Bcontrol 1 get abspoint /Ycont exch def /Xcont exch def Bcontrol 0 2 copy get 2 mul put Bcontrol 1 2 copy get 2 mul put Bcontrol Blen 2 sub 2 copy get 2 mul put Bcontrol Blen 1 sub 2 copy get 2 mul put /Ybi /Xbi currentpoint 3 1 roll def def 0 2 Blen 4 sub {/i exch def Bcontrol i get 3 div Bcontrol i 1 add get 3 div Bcontrol i get 3 mul Bcontrol i 2 add get add 6 div Bcontrol i 1 add get 3 mul Bcontrol i 3 add get add 6 div /Xbi Xcont Bcontrol i 2 add get 2 div add def /Ybi Ycont Bcontrol i 3 add get 2 div add def /Xcont Xcont Bcontrol i 2 add get add def /Ycont Ycont Bcontrol i 3 add get add def Xbi currentpoint pop sub Ybi currentpoint exch pop sub rcurveto }for dstroke}if}def % Psfig additions % startFig modified by WTR on 1/89 and further by denise 4/89 /startFig {/SavedState save def userdict maxlength dict begin currentpoint translate /ury exch def /urx exch def /lly exch def /llx exch def /y exch def /x exch def /sx x urx llx sub div def /sy y lly ury sub div def sx sy scale llx neg ury neg translate /DefFigCTM matrix currentmatrix def /initmatrix {DefFigCTM setmatrix} def /defaultmatrix {DefFigCTM exch copy} def /initgraphics {DefFigCTM setmatrix newpath 1 setlinewidth 0 setlinecap 0 setlinejoin [] 0 setdash 0 setgray 10 setmiterlimit}def /showpage {initgraphics} def } def % Args are llx lly urx ury (in figure coordinates) /clipFig { currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll exch lineto exch lineto exch lineto closepath clip newpath moveto} def % doclip, if called, will always be just after a `startfig' /doclip {llx lly urx ury clipFig} def /endFig {end SavedState restore} def /globalstart { % Push details about the enviornment on the stack. fontnum fontsize fontslant fontheight firstpage mh my resolution slotno currentpoint pagesave restore gsave } def /globalend { grestore moveto /slotno exch def /resolution exch def /my exch def /mh exch def /firstpage exch def /fontheight exch def /fontslant exch def /fontsize exch def /fontnum exch def F /pagesave save def } def % End Psfig additions end /ditstart{$DITroff begin /nfonts 60 def % NFONTS makedev/ditroff dependent! /fonts[nfonts{0}repeat]def /fontnames[nfonts{()}repeat]def /docsave save def }def % character outcalls /oc {/pswid exch def /cc exch def /name exch def /ditwid pswid fontsize mul resolution mul 72000 div def /ditsiz fontsize resolution mul 72 div def ocprocs name known{ocprocs name get exec}{name cb} ifelse}def /fractm [.65 0 0 .6 0 0] def /fraction {/fden exch def /fnum exch def gsave /cf currentfont def cf fractm makefont setfont 0 .3 dm 2 copy neg rmoveto fnum show rmoveto currentfont cf setfont(\244)show setfont fden show grestore ditwid 0 rmoveto} def /oce {grestore ditwid 0 rmoveto}def /dm {ditsiz mul}def /ocprocs 50 dict def ocprocs begin (14){(1)(4)fraction}def (12){(1)(2)fraction}def (34){(3)(4)fraction}def (13){(1)(3)fraction}def (23){(2)(3)fraction}def (18){(1)(8)fraction}def (38){(3)(8)fraction}def (58){(5)(8)fraction}def (78){(7)(8)fraction}def (sr){gsave .05 dm .16 dm rmoveto(\326)show oce}def (is){gsave 0 .15 dm rmoveto(\362)show oce}def (->){gsave 0 .02 dm rmoveto(\256)show oce}def (<-){gsave 0 .02 dm rmoveto(\254)show oce}def (==){gsave 0 .05 dm rmoveto(\272)show oce}def end % DIThacks fonts for some special chars %%DocumentSuppliedFonts: DIThacks %%BeginFont: DIThacks 50 dict dup begin /FontType 3 def /FontName /DIThacks def /FontMatrix [.001 0.0 0.0 .001 0.0 0.0] def /FontBBox [-220 -280 900 900] def% a lie but ... /Encoding 256 array def 0 1 255{Encoding exch /.notdef put}for Encoding dup 8#040/space put %space dup 8#110/rc put %right ceil dup 8#111/lt put %left top curl dup 8#112/bv put %bold vert dup 8#113/lk put %left mid curl dup 8#114/lb put %left bot curl dup 8#115/rt put %right top curl dup 8#116/rk put %right mid curl dup 8#117/rb put %right bot curl dup 8#120/rf put %right floor dup 8#121/lf put %left floor dup 8#122/lc put %left ceil dup 8#140/sq put %square dup 8#141/bx put %box dup 8#142/ci put %circle dup 8#143/br put %box rule dup 8#144/rn put %root extender dup 8#145/vr put %vertical rule dup 8#146/ob put %outline bullet dup 8#147/bu put %bullet dup 8#150/ru put %rule dup 8#151/ul put %underline pop /DITfd 100 dict def /BuildChar{0 begin /cc exch def /fd exch def /charname fd /Encoding get cc get def /charwid fd /Metrics get charname get def /charproc fd /CharProcs get charname get def charwid 0 fd /FontBBox get aload pop setcachedevice 40 setlinewidth newpath 0 0 moveto gsave charproc grestore end}def /BuildChar load 0 DITfd put %/UniqueID 5 def /CharProcs 50 dict def CharProcs begin /space{}def /.notdef{}def /ru{500 0 rls}def /rn{0 750 moveto 500 0 rls}def /vr{20 800 moveto 0 -770 rls}def /bv{20 800 moveto 0 -1000 rls}def /br{20 770 moveto 0 -1040 rls}def /ul{0 -250 moveto 500 0 rls}def /ob{200 250 rmoveto currentpoint newpath 200 0 360 arc closepath stroke}def /bu{200 250 rmoveto currentpoint newpath 200 0 360 arc closepath fill}def /sq{80 0 rmoveto currentpoint dround newpath moveto 640 0 rlineto 0 640 rlineto -640 0 rlineto closepath stroke}def /bx{80 0 rmoveto currentpoint dround newpath moveto 640 0 rlineto 0 640 rlineto -640 0 rlineto closepath fill}def /ci{355 333 rmoveto currentpoint newpath 333 0 360 arc 50 setlinewidth stroke}def /lt{20 -200 moveto 0 550 rlineto currx 800 2cx s4 add exch s4 a4p stroke}def /lb{20 800 moveto 0 -550 rlineto currx -200 2cx s4 add exch s4 a4p stroke}def /rt{20 -200 moveto 0 550 rlineto currx 800 2cx s4 sub exch s4 a4p stroke}def /rb{20 800 moveto 0 -500 rlineto currx -200 2cx s4 sub exch s4 a4p stroke}def /lk{20 800 moveto 20 300 -280 300 s4 arcto pop pop 1000 sub currentpoint stroke moveto 20 300 4 2 roll s4 a4p 20 -200 lineto stroke}def /rk{20 800 moveto 20 300 320 300 s4 arcto pop pop 1000 sub currentpoint stroke moveto 20 300 4 2 roll s4 a4p 20 -200 lineto stroke}def /lf{20 800 moveto 0 -1000 rlineto s4 0 rls}def /rf{20 800 moveto 0 -1000 rlineto s4 neg 0 rls}def /lc{20 -200 moveto 0 1000 rlineto s4 0 rls}def /rc{20 -200 moveto 0 1000 rlineto s4 neg 0 rls}def end /Metrics 50 dict def Metrics begin /.notdef 0 def /space 500 def /ru 500 def /br 0 def /lt 250 def /lb 250 def /rt 250 def /rb 250 def /lk 250 def /rk 250 def /rc 250 def /lc 250 def /rf 250 def /lf 250 def /bv 250 def /ob 350 def /bu 350 def /ci 750 def /bx 750 def /sq 750 def /rn 500 def /ul 500 def /vr 0 def end DITfd begin /s2 500 def /s4 250 def /s3 333 def /a4p{arcto pop pop pop pop}def /2cx{2 copy exch}def /rls{rlineto stroke}def /currx{currentpoint pop}def /dround{transform round exch round exch itransform} def end end /DIThacks exch definefont pop %%EndFont ditstart (psc)xT 576 1 1 xr 1(Times-Roman)xf 2(Times-Italic)xf 3(Times-Bold)xf 4(Times-BoldItalic)xf 5(Helvetica)xf 6(Helvetica-Bold)xf 7(Courier)xf 8(Courier-Bold)xf 9(Symbol)xf 10(DIThacks)xf 10 s 1 f xi %%EndProlog %%Page: 1 1 bp 10 s 0 xH 0 xS 1 f 32(--)Y 4323(--)X 3 f 13 s 1190 984(Inductive)N 1639(Reasoning)X 2128(and)X 2322(Kolmogorov)X 2903(Complexity)X 2 f 10 s 2156 1260(Ming)N 2345(Li*)X 1 f 1374 1428(University)N 1732(of)X 1819(Waterloo,)X 2158(Department)X 2557(of)X 2644(Computer)X 2984(Science)X 1709 1548(Waterloo,)N 2048(Ontario)X 2313(N2L)X 2480(3G1,)X 2658(Canada)X 2 f 2015 1764(Paul)N 2186(M.B.)X 2362(Vit)X 2469(\302)X 2455(anyi)X 1 f 1406 1932(Centrum)N 1706(voor)X 1873(Wiskunde)X 2218(en)X 2314(Informatica,)X 2729(Kruislaan)X 3061(413,)X 1693 2052(1098)N 1873(SJ)X 1968(Amsterdam,)X 2382(The)X 2527(Netherlands)X 2246 2172(and)N 1272 2292(Universiteit)N 1670(van)X 1806(Amsterdam,)X 2220(Faculteit)X 2520(Wiskunde)X 2865(en)X 2961(Informatica)X 11 s 2074 2928(ABSTRACT)N 864 3232(This)N 1047(is)X 1132(a)X 1198(sloppy)X 1460(\256rst)X 1624(draft)X 1817(of)X 1917([J.)X 2029(Comp.)X 2316(System)X 2603(Sciences,)X 2960(44:2\(1992\),)X 3400(343-384].)X 864 3376(Also,)N 1099(there)X 1322(are)X 1476(some)X 1709(problems)X 2084(with)X 2288(the)X 2443(pictures)X 2769(and)X 2942(references)X 3348(due)X 3521(to)X 3636(the)X 864 3520(obsolete)N 1180(troff)X 1358(processing.)X 864 3868(Reasoning)N 1261(to)X 1355(obtain)X 1601(the)X 1735(`truth')X 1986(about)X 2208(reality,)X 2482(from)X 2679(external)X 2989(data,)X 3184(is)X 3269(an)X 3378(important,)X 864 4028(controversial,)N 1377(and)X 1535(complicated)X 1998(issue)X 2205(in)X 2305(man's)X 2551(effort)X 2777(to)X 2877(understand)X 3294(nature.)X 3566(\(Yet,)X 864 4188(today,)N 1110(we)X 1240(try)X 1366(to)X 1463(make)X 1682(machines)X 2043(do)X 2159(this.\))X 2366(There)X 2599(have)X 2793(been)X 2988(old)X 3130(useful)X 3374(principles,)X 864 4348(new)N 1050(exciting)X 1375(models,)X 1692(and)X 1859(intricate)X 2189(theories)X 2508(scattered)X 2861(in)X 2970(vastly)X 3221(different)X 3564(areas)X 864 4508(including)N 1236(philosophy)X 1666(of)X 1777(science,)X 2096(statistics,)X 2461(computer)X 2832(science,)X 3151(and)X 3316(psychology.)X 864 4668(We)N 1013(focus)X 1230(on)X 1345(inductive)X 1701(reasoning)X 2070(in)X 2166(correspondence)X 2744(with)X 2927(ideas)X 3134(of)X 3233(R.J.)X 3396(Solomon-)X 864 4828(off.)N 1019(While)X 1266(his)X 1400(proposals)X 1768(result)X 1995(in)X 2095(perfect)X 2371(procedures,)X 2810(they)X 2994(involve)X 3291(the)X 3431(noncom-)X 864 4988(putable)N 1156(notion)X 1414(of)X 1518(Kolmogorov)X 1999(complexity.)X 2472(In)X 2576(this)X 2735(paper)X 2961(we)X 3094(develop)X 3404(the)X 3543(thesis)X 864 5148(that)N 1023(Solomonoff's)X 1533(method)X 1824(is)X 1909(fundamental)X 2376(in)X 2471(the)X 2605(sense)X 2821(that)X 2980(many)X 3202(other)X 3410(induction)X 864 5308(principles)N 1236(can)X 1381(be)X 1487(viewed)X 1764(as)X 1860(particular)X 2221(ways)X 2424(to)X 2516(obtain)X 2760(computable)X 3195(approximations)X 864 5468(to)N 961(it.)X 1083(We)X 1233(demonstrate)X 1692(this)X 1848(explicitly)X 2211(in)X 2308(the)X 2444(cases)X 2657(of)X 2758(Gold's)X 3026(paradigm)X 3388(for)X 3519(induc-)X 864 5628(tive)N 1021(inference,)X 1394(Rissanen's)X 1799(minimum)X 2167(description)X 2582(length)X 2826(\(MDL\))X 3102(principle,)X 3461(Fisher's)X 864 5788(maximum)N 1266(likelihood)X 1668(principle,)X 2048(and)X 2219(Jaynes')X 2526(maximum)X 2929(entropy)X 3242(principle.)X 3622(We)X ep %%Page: 2 2 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2231 424(-)N 2282(2)X 2348(-)X 864 736(present)N 1158(several)X 1447(new)X 1633(theorems)X 1995(and)X 2161(derivations)X 2592(to)X 2700(this)X 2867(effect.)X 3150(We)X 3311(also)X 3492(delimit)X 864 896(what)N 1065(can)X 1217(be)X 1331(learned)X 1621(and)X 1779(what)X 1981(cannot)X 2247(be)X 2361(learned)X 2651(in)X 2751(terms)X 2978(of)X 3082(Kolmogorov)X 3563(com-)X 864 1056(plexity,)N 1158(and)X 1311(we)X 1439(describe)X 1758(an)X 1866(experiment)X 2288(in)X 2382(machine)X 2706(learning)X 3020(of)X 3118(handwritten)X 3564(char-)X 864 1216(acters.)N 1122(We)X 1275(also)X 1448(give)X 1631(an)X 1745(application)X 2169(of)X 2273(Kolmogorov)X 2754(complexity)X 3184(in)X 3285(Valiant)X 3577(style)X 864 1376(learning,)N 1214(where)X 1467(we)X 1608(want)X 1818(to)X 1926(learn)X 2141(a)X 2219(concept)X 2532(probably)X 2884(approximally)X 3393(correct)X 3675(in)X 864 1536(feasible)N 1160(time)X 1340(and)X 1489(examples.)X 2 f 776 1884(The)N 936(eye)X 1081(of)X 1178(the)X 1314(understanding)X 1851(is)X 1938(like)X 2094(the)X 2230(eye)X 2375(of)X 2472(the)X 2608(sense;)X 2856(for)X 2988(as)X 3095(you)X 3251(may)X 3426(see)X 3567(great)X 3782(objects)X 576 2044(through)N 882(small)X 1100(crannies)X 1430(or)X 1535(levels,)X 1785(so)X 1890(you)X 2044(may)X 2217(see)X 2356(great)X 2569(axioms)X 2845(of)X 2940(nature)X 3196(through)X 3501(small)X 3718(and)X 3876(con-)X 576 2204(temptible)N 927(instances.)X 1 f 1299([Francis)X 1609(Bacon,)X 2 f 1878(Sylva)X 2091(Sylvarum)X 1 f 2445(337,)X 2621(1627])X 3 f 576 2524(1.)N 686(A)X 771(Historical)X 1166(View)X 1378(of)X 1473(Inductive)X 1852(Reasoning)X 1 f 576 2712(The)N 747(Oxford)X 1034(English)X 1338(Dictionary)X 1749(gives)X 1969(as)X 2077(the)X 2220(meaning)X 2559(of)X 3 f 2667(induction:)X 2 f 3089(the)X 3232(process)X 3535(of)X 3639(inferring)X 3988(a)X 576 2872(general)N 880(law)X 1043(or)X 1156(principle)X 1510(from)X 1711(the)X 1854(observations)X 2339(of)X 2442(particular)X 2834(instances.)X 1 f 3240(This)X 3431(de\256nes)X 3714(precisely)X 576 3032(what)N 770(we)X 895(would)X 1138(like)X 1294(to)X 1386(call)X 2 f 1537(inductive)X 1884(inference)X 1 f 2212(.)X 2279(On)X 2409(the)X 2540(other)X 2744(hand,)X 2960(we)X 3085(regard)X 2 f 3332(inductive)X 3680(reasoning)X 1 f 576 3192(as)N 676(a)X 742(more)X 950(general)X 1236(concept)X 1536(than)X 1714(inductive)X 2069(inference,)X 2444(namely)X 2730(as)X 2829(a)X 2894(process)X 3183(of)X 3282(re-assigning)X 3738(a)X 3803(proba-)X 576 3352(bility)N 788(\(or)X 914(credibility\))X 1331(to)X 1424(a)X 1487(law)X 1638(or)X 1735(proposition)X 2161(from)X 2356(the)X 2488(observation)X 2923(of)X 3020(particular)X 3383(instances.)X 3753(In)X 3851(other)X 576 3512(words,)N 852(in)X 961(the)X 1109(way)X 1295(we)X 1437(use)X 1594(the)X 1742(notions,)X 2064(inductive)X 2433(inference)X 2800(draws)X 3049(conclusions)X 3505(that)X 3678(consist)X 3963(in)X 2 f 576 3672(accepting)N 951(or)X 1061(rejecting)X 1 f 1407(a)X 1478(proposition,)X 1934(while)X 2162(inductive)X 2523(reasoning)X 2897(only)X 3086(changes)X 3401(the)X 3542(degree)X 3809(of)X 3915(our)X 576 3832(belief)N 804(in)X 899(a)X 964(proposition.)X 1414(The)X 1577(former)X 1842(is)X 1927(a)X 1992(special)X 2263(case)X 2440(of)X 2539(the)X 2673(latter.)X 2925(In)X 3024(this)X 3178(paper)X 3399(we)X 3527(discuss)X 3807(induc-)X 576 3992(tive)N 733(reasoning)X 1099(in)X 1192(correspondence)X 1767(with)X 1948(R.J.)X 2109(Solomonoff's)X 2617(ideas)X 2822(as)X 2919(expressed)X 3289(in)X 3383(e.g.)X 3557([)X 3611(Solomonoff)X 576 4152(1964)N 790(However,)X 1171(Solomonoff's)X 1693(procedure)X 2082(is)X 2179(not)X 2330(effective,)X 2698(since)X 2917(it)X 3005(involves)X 3342(the)X 3488(noncomputable)X 576 4312(Kolmogorov)N 1051(complexity)X 1474(of)X 1572(objects.)X 1891(We)X 2038(shall)X 2230(show,)X 2462(however,)X 2811(that)X 2969(there)X 3170(is)X 3254(considerable)X 3724(structure)X 576 4472(in)N 669(many)X 889(different)X 1216(approaches)X 1635(proposed)X 1981(for)X 2107(induction,)X 2487(since)X 2691(they)X 2866(can)X 3011(be)X 3117(variously)X 3468(derived)X 3755(as)X 3851(com-)X 576 4632(putable)N 858(approximations)X 1429(to)X 1520(Solomonoff's)X 2026(method.)X 776 4820(The)N 938(history)X 1208(of)X 1306(inductive)X 1661(inference,)X 2036(which)X 2277(is)X 2362(as)X 2461(old)X 2600(as)X 2699(empirical)X 3059(science)X 3344(itself,)X 3569(dates)X 3776(back)X 3968(at)X 576 4980(least)N 765(to)X 861(the)X 996(Greek)X 1237(philosopher)X 1680(of)X 1779(science)X 2064(Epicurus)X 2403(\(342?)X 2629(-)X 2684(270?)X 2881(B.C\).)X 3120(To)X 3244(reason)X 3499(by)X 3613(induction)X 3973(is)X 576 5140(nothing)N 873(but)X 1013(to)X 1109(learn)X 1312(from)X 1510(experience.)X 1940(As)X 2064(the)X 2199(sun)X 2348(rises)X 2536(day)X 2690(by)X 2805(day,)X 2981(our)X 3125(belief)X 3353(in)X 3449(that)X 3609(the)X 3744(sun)X 3894(will)X 9 s 10 f 576 5228(hhhhhhhhhhhhhhhh)N 1 f 576 5344(*)N 636(The)X 772(work)X 944(of)X 1028(the)X 1140(\256rst)X 1276(author)X 1484(was)X 1620(supported)X 1928(in)X 2008(part)X 2144(by)X 2240(National)X 2512(Science)X 2760(Foundation)X 3112(Grant)X 3300(DCR-8606366,)X 3767(Of\256ce)X 3972(of)X 576 5432(Naval)N 771(Research)X 1058(Grant)X 1245(N00014-85-k-0445,)X 1842(Army)X 2033(Research)X 2320(Of\256ce)X 2523(Grant)X 2710(DAAL03-86-K-0171,)X 3363(and)X 3490(NSERC)X 3744(Operating)X 576 5520(Grant)N 768(OGP0036747.)X 1228(Part)X 1372(of)X 1460(the)X 1576(work)X 1752(was)X 1892(performed)X 2220(while)X 2408(he)X 2504(was)X 2644(at)X 2724(the)X 2840(Department)X 3208(of)X 3296(Computer)X 3613(Science,)X 3884(York)X 576 5608(University,)N 922(North)X 1114(York,)X 1304(Ontario)X 1548(,)X 1590(Canada.)X 1848(A)X 1924(preliminary)X 2284(form)X 2448(of)X 2532(this)X 2660(paper)X 2844(appeared)X 3128(in)X 2 f 3208(Proc.)X 3390(4th)X 3506(IEEE)X 3685(Structure)X 3976(in)X 576 5696(Complexity)N 922(Theory)X 1144(Conference)X 1 f 1476(,)X 1512(1989.)X ep %%Page: 3 3 bp 9 s 0 xH 0 xS 1 f 11 s 10 s 0 32(--)N 4323(--)X 11 s 2231 424(-)N 2282(3)X 2348(-)X 576 736(rise)N 733(tomorrow)X 1110(increases,)X 1484(and)X 1641(we)X 1773(eventually)X 2171(infer)X 2367(the)X 2 f 2505(truth)X 1 f 2707(that)X 2870(the)X 3009(sun)X 2 f 3162(will)X 1 f 3327(rise)X 3485(every)X 3711(morning.)X 576 896(As)N 707(human)X 981(history)X 1260(evolves,)X 1585(man)X 1771(tries)X 1957(to)X 2060(understand)X 2480(and)X 2641(explain)X 2935(the)X 3077(events)X 3335(that)X 3501(happen)X 3788(around)X 576 1056(him:)N 763(this)X 915(takes)X 1120(the)X 1252(form)X 1447(of)X 1544(different)X 1871(induction)X 2229(methods)X 2552(to)X 2645(formulate)X 3012(scienti\256c)X 3356(theories)X 3660(from)X 3856(posi-)X 576 1216(tive)N 746(and)X 910(negative,)X 1268(fortunate)X 1623(and)X 1787(unfortunate,)X 2252(lucky)X 2485(and)X 2648(unlucky,)X 2990(happy)X 3241(and)X 3404(miserable)X 3783(experi-)X 576 1376(ences.)N 837(Two)X 1020(metaphysical)X 1508(principles)X 1879(stand)X 2088(out)X 2224(and)X 2374(prevail)X 2642(today:)X 2886(the)X 3017(principle)X 3354(of)X 3450(Epicurus')X 3815(multi-)X 576 1536(ple)N 732(explanations)X 1226(\(or)X 1375(indifference\))X 1876(and)X 2050(Occam's)X 2409(principle)X 2770(of)X 2890(simplest)X 3232(explanation)X 3691(\(Occam's)X 576 1696(razor\).)N 776 1884(The)N 3 f 946(Principle)X 1323(of)X 1429(Multiple)X 1786(Explanations:)X 2 f 2344(If)X 2432(more)X 2646(than)X 2837(one)X 2998(theory)X 3257(is)X 3350(consistent)X 3737(with)X 3924(the)X 576 2044(data,)N 777(keep)X 960(them)X 1153(all.)X 1 f 576 2232(The)N 741(source)X 998(of)X 1099(the)X 1235(following)X 1606(material)X 1924(is)X 2012([)X 2070(Epicurus)X 2412(Epicurus,)X 2776(in)X 2874(his)X 2 f 3006(Letter)X 3246(to)X 3344(Pythocles)X 1 f 3687(,)X 3738(explains)X 576 2392(that:)N 773(There)X 1017(are)X 1163(cases,)X 1409(especially)X 1801(of)X 1913(events)X 2177(in)X 2285(the)X 2432(heavens)X 2754(such)X 2954(as)X 3066(the)X 3213(risings)X 3486(and)X 3651(settings)X 3959(of)X 576 2552(heavenly)N 919(bodies)X 1174(and)X 1326(eclipses,)X 1652(where)X 1891(it)X 1966(is)X 2050(suf\256cient)X 2403(for)X 2530(our)X 2672(happiness)X 3044(that)X 3202(several)X 3477(explanations)X 3949(be)X 576 2712(discovered.)N 1006(In)X 1106(these)X 1314(cases,)X 1548(the)X 1683(events)X 1935(`have)X 2157(multiple)X 2480(causes)X 2736(of)X 2836(coming)X 3128(into)X 3293(being)X 3516(and)X 3670(a)X 3736(multiple)X 576 2872(predication)N 995(of)X 1090(what)X 1284(exists,)X 1530(in)X 1622(agreement)X 2012(with)X 2192(the)X 2323(perceptions.')X 2825(Epicurus)X 3161(maintains)X 3528(that,)X 3706(if)X 3783(several)X 576 3032(explanations)N 1047(are)X 1179(in)X 1273(agreement)X 1665(with)X 1847(the)X 1979(\(heavenly\))X 2379(phenomena,)X 2831(then)X 3007(we)X 3133(must)X 3329(keep)X 3519(all)X 3632(of)X 3729(them)X 3930(for)X 576 3192(two)N 740(reasons.)X 1057(Firstly,)X 1342(the)X 1482(degree)X 1748(of)X 1853(precision)X 2208(achieved)X 2553(by)X 2673(multiple)X 3001(explanations)X 3479(is)X 3570(suf\256cient)X 3930(for)X 576 3352(human)N 847(happiness.)X 1247(Secondly,)X 1628(it)X 1709(would)X 1960(be)X 2074(unscienti\256c)X 2512(to)X 2611(prefer)X 2850(one)X 3007(explanation)X 3449(to)X 3548(another)X 3842(when)X 576 3512(both)N 757(are)X 888(equally)X 1172(in)X 1265(agreement)X 1656(with)X 1837(the)X 1970(phenomena.)X 2423(This,)X 2627(he)X 2735(claims,)X 3013(would)X 3258(be)X 3366(to)X 3460(`abandon)X 3812(physi-)X 576 3672(cal)N 704(inquiry)X 984(and)X 1136(resort)X 1361(to)X 1455(myth.')X 1735(His)X 1881(follower)X 2203(Lucretius)X 2560(\(95)X 2701(-)X 2754(55)X 2866(B.C.\))X 3103(illustrates)X 3471(the)X 3603(inevitabili)X 3963(ty)X 576 3832(of)N 671(the)X 801(use)X 940(of)X 1035(the)X 1165(multiple)X 1483(explanation)X 1917(principle)X 2253(by)X 2363(the)X 2493(following)X 2858(example:)X 10 s 776 4004(``There)N 1041(are)X 1163(also)X 1315(some)X 1507(things)X 1725(for)X 1842(which)X 2061(it)X 2129(is)X 2206(not)X 2332(enough)X 2592(to)X 2678(state)X 2849(a)X 2909(single)X 3124(cause,)X 3347(but)X 3473(several,)X 3745(of)X 3836(which)X 576 4148(one,)N 734(however,)X 1053(is)X 1128(the)X 1248(case.)X 1429(Just)X 1575(as)X 1664(if)X 1735(you)X 1876(were)X 2054(to)X 2137(see)X 2261(the)X 2380(lifeless)X 2628(corpse)X 2859(of)X 2947(a)X 3004(man)X 3163(lying)X 3348(far)X 3459(away,)X 3670(it)X 3735(would)X 3956(be)X 576 4292(\256tting)N 793(to)X 882(state)X 1056(all)X 1163(the)X 1288(causes)X 1525(of)X 1619(death)X 1820(in)X 1910(order)X 2108(that)X 2256(the)X 2382(single)X 2601(cause)X 2808(of)X 2903(this)X 3046(death)X 3248(may)X 3414(be)X 3518(stated.)X 3773(For)X 3912(you)X 576 4436(would)N 799(not)X 924(be)X 1023(able)X 1180(to)X 1265(establish)X 1568(conclusively)X 1996(that)X 2139(he)X 2238(died)X 2399(by)X 2502(the)X 2623(sword)X 2842(or)X 2932(of)X 3022(cold)X 3183(or)X 3273(of)X 3363(illness)X 3590(or)X 3680(perhaps)X 3952(by)X 576 4580(poison,)N 829(but)X 951(we)X 1065(know)X 1263(that)X 1403(there)X 1584(is)X 1657(something)X 2010(of)X 2097(this)X 2232(kind)X 2394(that)X 2534(happened)X 2862(to)X 2944(him.'')X 11 s 776 4768(Based)N 1015(on)X 1127(the)X 1259(same)X 1464(intuition,)X 1812(in)X 1906(the)X 2039(calculus)X 2353(of)X 2451(probabilities)X 2918(it)X 2993(has)X 3135(been)X 3326(customary)X 3718(to)X 3812(postu-)X 576 4928(late)N 727(the)X 858(`principle)X 1224(of)X 1320(indifference')X 1797(or)X 1893(the)X 2024(`principle)X 2390(of)X 2486(insuf\256cient)X 2906(reason'.)X 3230(When)X 3462(there)X 3660(is)X 3741(no)X 3851(other)X 576 5088(evidence,)N 934(because)X 1235(of)X 1331(the)X 1463(absolute)X 1781(lack)X 1952(of)X 2049(knowledge)X 2459(concerning)X 2874(the)X 3006(conditions)X 3398(under)X 3622(which)X 3861(a)X 3924(die)X 576 5248(falls,)N 774(we)X 900(have)X 1090(no)X 1202(reason)X 1455(to)X 1548(assume)X 1830(that)X 1986(a)X 2048(certain)X 2311(face)X 2480(has)X 2620(higher)X 2868(probability)X 3279(of)X 3375(turning)X 3653(up.)X 3808(Hence)X 576 5408(we)N 700(assume)X 981(that)X 1136(each)X 1319(side)X 1484(of)X 1580(the)X 1711(die)X 1842(has)X 1982(the)X 2113(probability)X 2524(1/6.)X 2682(The)X 3 f 2842(principle)X 3204(of)X 3300(indifference)X 1 f 3773(consid-)X 576 5568(ers)N 706(events)X 959(to)X 1056(be)X 1167(equally)X 1454(probable)X 1789(if)X 1870(we)X 1999(have)X 2192(not)X 2332(the)X 2467(slightest)X 2789(knowledge)X 3202(of)X 3302(the)X 3437(conditions)X 3832(under)X 576 5728(which)N 819(each)X 1008(of)X 1109(them)X 1315(is)X 1403(going)X 1633(to)X 1731(occur.)X 1977(For)X 2128(the)X 2265(case)X 2445(of)X 2547(a)X 2615(die,)X 2774(this)X 2931(actually)X 3240(coincides)X 3602(with)X 3788(the)X 3925(so-)X 576 5888(called)N 820(`maximum)X 1241(entropy)X 1543(principle',)X 1941(we)X 2076(will)X 2247(discuss)X 2534(later,)X 2746(which)X 2994(states)X 3222(that)X 3387(we)X 3521(should)X 3788(choose)X ep %%Page: 4 4 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2231 424(-)N 2282(4)X 2348(-)X 576 754(probabilities)N 2 f 1041(p)X 8 s 771(i)Y 1 f 11 s 1133 754(for)N 1258(face)X 2 f 1427(i)X 1 f 1475(to)X 1567(be)X 1674(the)X 1806(outcome)X 2134(of)X 2231(a)X 2294(trial,)X 2 f 2483(i)X 9 f 2529(=)X 1 f 2591(1,)X 2678(2,)X 2751(...,)X 2846(6,)X 2936(such)X 3121(that)X 2 f 9 f 3278(-)X 1 f 16 s 9 f 3326 780(S)N 2 f 8 s 3407 798(i)N 9 f 3435(=)X 1 f 3470(1)X 3407 721(6)N 2 f 11 s 3509 754(p)N 8 s 771(i)Y 1 f 11 s 3592 754(ln)N 2 f 3675(p)X 8 s 771(i)Y 1 f 11 s 3768 754(is)N 3851(max-)X 576 932(imized)N 839(under)X 1061(the)X 1191(only)X 1370(constraint)X 16 s 9 f 1740 958(S)N 2 f 8 s 1821 976(i)N 9 f 1849(=)X 1 f 1884(1)X 1821 899(6)N 2 f 11 s 1923 932(p)N 8 s 949(i)Y 11 s 9 f 2006 932(=)N 1 f 2068(1.)X 2156(We)X 2300(obtain)X 2543(precisely)X 2 f 2883(p)X 8 s 949(i)Y 11 s 9 f 2966 932(=)N 1 f 3028(1)X 2 f 3079(/)X 1 f 3111(6)X 3177(for)X 2 f 3301(i)X 9 f 3347(=)X 1 f 3409(1,)X 3496(2,)X 3569(...,)X 3664(6.)X 776 1120(The)N 936(second)X 1203(and)X 1353(more)X 1557(sophisticated)X 2041(principle)X 2378(is)X 2460(the)X 2591(celebrated)X 2976(Occam's)X 3311(razor)X 3514(principle)X 3851(com-)X 576 1280(monly)N 827(attributed)X 1191(to)X 1285(William)X 1601(of)X 1699(Ockham)X 2022(\(1290?)X 2291(-)X 2345(1349?\).)X 2658(This)X 2840(enters)X 3075(the)X 3208(scene)X 3427(about)X 3647(1500)X 3847(years)X 576 1440(after)N 759(Epicurus.)X 1138(In)X 1233(sharp)X 1445(contrast)X 1746(to)X 1837(the)X 1967(principle)X 2303(of)X 2398(multiple)X 2716(explanations,)X 3206(it)X 3278(states:)X 3 f 776 1628(Occam's)N 1125(Razor)X 1376(Principle:)X 2 f 1771(Entities)X 2064(should)X 2321(not)X 2456(be)X 2561(multiplied)X 2942(beyond)X 3218(necessity.)X 1 f 776 1816(This)N 960(is)X 1046(generally)X 1401(interpreted)X 1810(as:)X 1935(Among)X 2226(the)X 2361(several)X 2638(theories)X 2945(that)X 3106(are)X 3241(all)X 3358(consistent)X 3739(with)X 3924(the)X 576 1976(observed)N 923(phenomena,)X 1381(one)X 1538(should)X 1803(pick)X 1985(the)X 2123(simplest)X 2448(theory.)X 2747(\(According)X 3177(to)X 3276(Bertrand)X 3613(Russell,)X 3924(the)X 576 2136(actual)N 816(phrase)X 1074(used)X 1264(by)X 1381(Ockham)X 1709(was:)X 1900(`It)X 2013(is)X 2102(vain)X 2284(to)X 2383(do)X 2501(with)X 2688(more)X 2899(what)X 3100(can)X 3252(be)X 3365(done)X 3566(with)X 3753(fewer.'\))X 576 2296(Surely)N 838(Occam's)X 1182(razor)X 1394(principle)X 1740(is)X 1831(easily)X 2069(understood)X 2492(from)X 2695(an)X 2810(`utilitarian')X 3245(point)X 3459(of)X 3563(view:)X 3790(if)X 3875(both)X 576 2456(theories)N 882(explain)X 1169(the)X 1304(same)X 1512(set)X 1637(of)X 1737(facts,)X 1952(why)X 2130(not)X 2270(use)X 2415(the)X 2551(simpler)X 2844(theory?!)X 3165(However)X 3514(things)X 3758(become)X 576 2616(more)N 791(intricate)X 1115(when)X 1339(we)X 1475(want)X 1680(to)X 1783(know)X 2012(whether)X 2329(a)X 2402(simpler)X 2701(theory)X 2960(is)X 3053(really)X 3288(better)X 3523(than)X 3709(the)X 3851(more)X 576 2776(complicated)N 1031(one.)X 1225(This)X 1405(also)X 1570(raises)X 1793(another)X 2080(question)X 2402(which)X 2640(has)X 2781(been)X 2971(a)X 3034(bone)X 3229(of)X 3326(contention)X 3723(in)X 3816(Philo-)X 576 2936(sophy)N 815(ever)X 995(since)X 1205(the)X 1342(razor's)X 1614(inception.)X 2016(For)X 2167(what)X 2367(is)X 2455(the)X 2592(proper)X 2850(measure)X 3172(of)X 3274(simplicit)X 3585(y?)X 3697(Is)X 2 f 3788(x)X 1 f 8 s 3837 2901(100)N 2 f 11 s 9 f 3940 2936(+)N 1 f 3988(1)X 576 3096(more)N 793(complicated)X 1261(than)X 2 f 1449(ax)X 1 f 8 s 1542 3061(17)N 2 f 11 s 9 f 1613 3096(+)N 2 f 1661(bx)X 1 f 8 s 1754 3061(2)N 2 f 11 s 9 f 1793 3096(+)N 2 f 1841(cx)X 9 f 1926(+)X 2 f 1974(d)X 1 f (?)S 2115(E.g.,)X 2315(the)X 2460(distinguished)X 2968(contemporary)X 3494(philosopher)X 3947(K.)X 576 3256(Popper)N 849(pronounced)X 1288(that)X 1445(the)X 1576(razor)X 1779(is)X 1861(without)X 2154(sense)X 2367(to)X 2459(use)X 2599(on)X 2710(such)X 2894(grounds.)X 3222(However,)X 3588(it)X 3661(is)X 3743(interest-)X 576 3416(ing)N 718(to)X 816(notice)X 1061(that)X 1223(the)X 1360(principle)X 1703(can)X 1854(be)X 1966(given)X 2191(objective)X 2544(contents,)X 2889(and)X 3045(has)X 3191(recently)X 3504(been)X 3700(very)X 3886(suc-)X 576 3576(cessfully)N 911(applied)X 1193(in)X 1284(many)X 1502(different)X 1827(forms)X 2054(in)X 2145(computational)X 2673(learning)X 2984(theory.)X 776 3764(To)N 903(explain)X 1192(this,)X 1371(let)X 1490(us)X 1598(consider)X 1926(an)X 2039(over-simpli\256ed)X 2608(example)X 2937(of)X 3040(inferring)X 3378(a)X 3447(\256nite)X 3659(automaton)X 576 3924(with)N 755(one-letter)X 1115(input)X 1319(using)X 1532(Occam's)X 1866(razor)X 2068(principle.)X 776 4112(Accepted)N 1130(inputs:)X 1393(0,)X 1481(000,)X 1657(00000,)X 1921(000000000;)X 776 4300(Rejected)N 1107(inputs:)X 9 f 1370(e)X 1 f 1409(,)X 1453(00,)X 1585(000000;)X 576 4488(For)N 725(these)X 933(data,)X 1129(there)X 1332(exist)X 1526(many)X 1749(consistent)X 2129(\256nite)X 2338(automata.)X 2711(Figure)X 2968(1)X 3039(shows)X 3285(the)X 3420(trivial)X 3659(automaton)X 576 4648(and)N 732(Figure)X 991(2)X 1064(shows)X 1312(the)X 1449(smallest)X 1768(automaton,)X 2192(where)X 2 f 2435(S)X 1 f 2508(denotes)X 2806(starting)X 3100(state)X 3291(and)X 3447(darker)X 3700(states)X 3925(are)X 576 4808(accepting)N 936(states.)X 1121 5349(S)N 1082 5332 MXY 124 Dc 1365 MX 124 Dc 1373 MX 107 Dc 1646 MX 124 Dc 1928 MX 124 Dc 1937 MX 107 Dc 2210 MX 124 Dc 2491 MX 124 Dc 2500 MX 107 Dc 2773 MX 124 Dc 3055 MX 124 Dc 3336 MX 124 Dc 3619 MX 124 Dc 3627 MX 107 Dc 1036 MX 47 0 Dl 1051 5340 MXY 31 -8 Dl 1051 5324 MXY 31 8 Dl 1264 5269(0)N 1208 5332 MXY 156 0 Dl 1333 5340 MXY 31 -8 Dl 1333 5324 MXY 31 8 Dl 1546 5269(0)N 1490 5332 MXY 156 0 Dl 1615 5340 MXY 31 -8 Dl 1615 5324 MXY 31 8 Dl 1828 5269(0)N 1771 5332 MXY 156 0 Dl 1896 5340 MXY 31 -8 Dl 1896 5324 MXY 31 8 Dl 2109 5269(0)N 2053 5332 MXY 156 0 Dl 2179 5340 MXY 31 -8 Dl 2179 5324 MXY 31 8 Dl 2391 5269(0)N 2335 5332 MXY 156 0 Dl 2460 5340 MXY 31 -8 Dl 2460 5324 MXY 31 8 Dl 2673 5269(0)N 2616 5332 MXY 156 0 Dl 2742 5340 MXY 31 -8 Dl 2742 5324 MXY 31 8 Dl 2954 5269(0)N 2899 5332 MXY 156 0 Dl 3024 5340 MXY 31 -8 Dl 3024 5324 MXY 31 8 Dl 3237 5269(0)N 3180 5332 MXY 156 0 Dl 3305 5340 MXY 31 -8 Dl 3305 5324 MXY 31 8 Dl 3518 5269(0)N 3462 5332 MXY 156 0 Dl 3587 5340 MXY 31 -8 Dl 3587 5324 MXY 31 8 Dl 1787 5537(Figure)N 2039(1:)X 2130(A)X 2215(trivial)X 2449(automaton)X ep %%Page: 5 5 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2231 424(-)N 2282(5)X 2348(-)X 2233 985(S)N 2164 968 MXY 186 Dc 2629 MX 186 Dc 2638 MX 167 Dc 2094 MX 69 0 Dl 2117 979 MXY 46 -11 Dl 2117 956 MXY 46 11 Dl 2586 957 MXY 47 -7 Dl 2588 934 MXY 45 15 Dl 2634 MX -144 1385 -144 -1385 Da 2393 979 MXY -47 7 Dl 2390 1002 MXY -45 -15 Dl 2346 986 MXY 144 -1385 144 1385 Da 2468 892(0)N 2468 1078(0)N 1711 1263(Figure)N 1963(2:)X 2054(The)X 2213(smallest)X 2525(automaton)X 776 1690(Intuitively,)N 1193(the)X 1327(automaton)X 1726(in)X 1821(Figure)X 2077(1)X 2147(just)X 2301(encodes)X 2610(data)X 2783(plainly,)X 3077(we)X 3205(therefore)X 3548(do)X 3662(not)X 3802(expect)X 576 1850(that)N 736(machine)X 1062(anticipate)X 1432(the)X 1566(future)X 1802(data.)X 1997(On)X 2130(the)X 2264(other)X 2471(hand)X 2668(the)X 2802(second)X 3072(machine)X 3397(makes)X 3648(a)X 3713(plausible)X 2 f 576 2010(inference)N 1 f 943(that)X 1115(the)X 1262(language)X 1619(accepted)X 1967(consists)X 2286(of)X 2399(strings)X 2674(of)X 2787(an)X 2910(odd)X 3082(number)X 3391(of)X 3504(0's.)X 3673(The)X 3850(latter)X 576 2170(appeals)N 882(to)X 993(our)X 1152(intuition)X 1495(as)X 1610(a)X 1691(reasonable)X 2109(inference.)X 2521(However,)X 2905(a)X 2985(too)X 3139(simplistic)X 3525(application)X 3959(of)X 576 2330(Occam's)N 910(razor)X 1112(principle)X 1448(may)X 1622(also)X 1786(lead)X 1955(to)X 2046(nonsense)X 2390(as)X 2485(the)X 2615(following)X 2980(story)X 3178(illustrates.)X 10 s 776 2582(Once)N 966(upon)X 1147(a)X 1204(time,)X 1387(there)X 1569(was)X 1715(a)X 1772(little)X 1939(girl)X 2071(named)X 2306(Emma.)X 2576(Emma)X 2806(had)X 2943(never)X 3143(eaten)X 3334(a)X 3391(banana,)X 3660(nor)X 3788(had)X 3925(she)X 576 2726(been)N 749(on)X 850(a)X 907(train.)X 1115(One)X 1270(day)X 1407(she)X 1535(went)X 1712(for)X 1826(a)X 1882(journey)X 2147(from)X 2323(New)X 2495(York)X 2680(to)X 2762(Pittsburgh)X 3110(by)X 3210(train.)X 3417(To)X 3526(relieve)X 3765(Emma's)X 576 2870(anxiety,)N 857(her)X 985(mother)X 1237(gave)X 1415(her)X 1544(a)X 1606(large)X 1793(bag)X 1935(of)X 2028(bananas.)X 2353(At)X 2459(Emma's)X 2752(\256rst)X 2902(bite)X 3048(of)X 3141(a)X 3203(banana,)X 3477(the)X 3601(train)X 3774(plunged)X 576 3014(into)N 722(a)X 780(tunnel.)X 1022(At)X 1124(the)X 1244(second)X 1489(bite,)X 1651(the)X 1771(train)X 1940(broke)X 2145(into)X 2291(daylight)X 2575(again.)X 2791(At)X 2893(the)X 3013(third)X 3186(bite,)X 3348(Lo!)X 3486(into)X 3632(a)X 3690(tunnel;)X 3934(the)X 576 3158(fourth)N 799(bite,)X 966(La!)X 1105(into)X 1256(daylight)X 1545(again.)X 1766(And)X 1932(so)X 2031(on)X 2139(all)X 2247(the)X 2373(way)X 2535(to)X 2625(Pittsburgh)X 2981(and)X 3125(to)X 3215(the)X 3341(bottom)X 3595(of)X 3690(her)X 3821(bag)X 3965(of)X 576 3302(bananas.)N 876(Our)X 1022(bright)X 1234(little)X 1401(Emma)X 1631(\(applying)X 1959(Occam's)X 2266(razor)X 2453(principle?\))X 2822(told)X 2966(her)X 3089(grandpa:)X 3390(`Every)X 3629(odd)X 3769(bite)X 3909(of)X 3996(a)X 576 3446(banana)N 833(makes)X 1067(you)X 1216(blind;)X 1432(every)X 1641(even)X 1823(bite)X 1973(puts)X 2136(things)X 2361(right)X 2542(again.')X 2813([After)X 3040(N.R.)X 3221(Hanson,)X 3516(`Perception)X 3916(and)X 576 3590(Discovery',)N 973(Freeman,)X 1294(Cooper)X 1550(&)X 1632(Co,)X 1765(1969,)X 1965(p.359.])X 11 s 776 3850(Let)N 928(us)X 1040(consider)X 1372(how)X 1557(the)X 1699(idea)X 1880(of)X 1987(`simplicity')X 2434(affects)X 2702(a)X 2775(scientist's)X 3162(thinking.)X 3535(We)X 3691(refer)X 3890(to)X 3993(a)X 576 4010(beautiful)N 913(study)X 1127(of)X 1223(simplicit)X 1534(y)X 1601(by)X 1712(Kemeny)X 2033([)X 2084(Kemeny)X 2404(Initially,)X 2729(there)X 2927(were)X 3119(no)X 3229(new)X 3397(facts)X 3585(that)X 3740(failed)X 3963(to)X 576 4170(be)N 688(explained)X 1060(by)X 1177(the)X 1314(Special)X 1603(Theory)X 1886(of)X 1988(relativity.)X 2359(The)X 2525(incentive)X 2879(to)X 2978(invent)X 3229(the)X 3367(General)X 3675(Theory)X 3959(of)X 576 4330(Relativity,)N 980(by)X 1100(Albert)X 1357(Einstein,)X 1701(was)X 1869(his)X 2004(conviction)X 2409(that)X 2574(the)X 2714(Special)X 3006(Theory)X 3292(was)X 3460(not)X 3604(the)X 2 f 3743(simplest)X 1 f 576 4490(theory)N 837(that)X 1006(can)X 1164(explain)X 1460(all)X 1586(the)X 1731(observed)X 2085(facts.)X 2310(Reducing)X 2685(the)X 2830(number)X 3136(of)X 3246(independent)X 3714(variables)X 576 4650(obviously)N 958(simpli\256es)X 1336(a)X 1409(theory.)X 1690(By)X 1827(the)X 1969(requirement)X 2429(of)X 2536(general)X 2829(covariance)X 3244(Einstein)X 3568(succeeded)X 3963(in)X 576 4810(replacing)N 926(the)X 1056(previous)X 1381(independent)X 1834(`gravitational)X 2332(mass')X 2559(and)X 2708(`inertial)X 3010(mass')X 3237(by)X 3347(a)X 3408(single)X 3641(concept.)X 776 4998(In)N 888(spite)X 1094(of)X 1206(the)X 1353(apparent)X 1695(universal)X 2057(acceptance)X 2482(of)X 2594(Occam's)X 2945(razor,)X 3186(consciously)X 3641(or)X 3754(uncons-)X 576 5158(ciously,)N 902(the)X 1059(concept)X 1382(of)X 1504(simplicit)X 1815(y)X 1908(remains)X 2235(highly)X 2509(controversial.)X 3039(Generally)X 3434(speaking,)X 3817(it)X 3915(has)X 576 5318(remained)N 926(a)X 987(crude)X 1204(non-precise)X 1637(idea.)X 1851(Things)X 2119(are)X 2249(subtler)X 2512(than)X 2687(they)X 2862(appear.)X 3141(Is)X 3227(the)X 3358(following)X 3724(formula-)X 576 5478(tion)N 736(precise?)X 3 f 776 5666(Occam's)N 1128(Razor)X 1382(Rule:)X 2 f 1612(Select)X 1848(a)X 1917(hypothesis)X 2314(which)X 2550(is)X 2634(as)X 2737(well)X 2910(in)X 3005(agreement)X 3402(with)X 3581(the)X 3715(observed)X 576 5826(value)N 789(as)X 889(possible;)X 1229(if)X 1301(there)X 1504(is)X 1585(any)X 1734(choice)X 1986(left,)X 2144(choose)X 2410(the)X 2540(simplest)X 2851(possible)X 3162(hypothesis.)X ep %%Page: 6 6 bp 11 s 0 xH 0 xS 2 f 1 f 10 s 0 32(--)N 4323(--)X 11 s 2231 424(-)N 2282(6)X 2348(-)X 3 f 776 736(Example.)N 1 f 1154(Consider)X 1495(the)X 1626(problem)X 1943(of)X 2039(\256tting)X 2 f 2274(n)X 1 f 2341(points)X 2580(by)X 2692(a)X 2755(polynomial.)X 3226(The)X 3387(above)X 3621(rule)X 3782(tells)X 3954(us)X 576 896(to)N 672(choose)X 943(the)X 1078(polynomial)X 1508(of)X 1608(lowest)X 1865(degree)X 2126(passing)X 2417(through)X 2718(all)X 2834(the)X 2 f 2969(n)X 1 f 3040(points.)X 3305(But)X 3460(due)X 3614(to)X 3710(measure-)X 576 1056(ment)N 786(precision)X 1142(and)X 1302(possible)X 1624(noise,)X 1865(whatever)X 2220(degree)X 2487(polynomial)X 2923(the)X 3064(points)X 3313(originated)X 3704(from,)X 3930(we)X 576 1216(will)N 743(end)X 899(up)X 1016(a)X 1084(polynomial)X 1516(of)X 1618(degree)X 2 f 1881(n)X 9 f 1939(-)X 1 f 1987(1)X 2059(which)X 2302(\256ts)X 2438(the)X 2574(data)X 2749(precisely.)X 3117(But)X 3273(this)X 3429(polynomial)X 3860(most)X 576 1376(likely)N 800(does)X 983(not)X 1118(help)X 1292(us)X 1392(to)X 1483(predict)X 1750(future)X 1982(points.)X 3 f 776 1564(Example)N 1 f 1109(.)X 1161(Consider)X 1509(another)X 1803(example)X 2132(given)X 2358(by)X 2477(Kemeny:)X 2831(Let)X 2980(there)X 3187(be)X 3301(unknown)X 3659(number)X 3959(of)X 576 1724(white)N 798(balls)X 991(and)X 1144(black)X 1361(balls)X 1554(in)X 1649(a)X 1714(sealed)X 1960(urn.)X 2147(Through)X 2476(an)X 2585(opening)X 2895(you)X 3053(randomly)X 3417(pick)X 3594(one)X 3746(ball)X 3904(at)X 3993(a)X 576 1884(time,)N 783(note)X 962(its)X 1073(color)X 1282(and)X 1437(replace)X 1719(it,)X 1819(and)X 1974(shake)X 2202(the)X 2338(urn)X 2483(thoroughly.)X 2920(After)X 2 f 3133(n)X 1 f 3205(draws)X 3442(you)X 3602(must)X 3802(decide)X 576 2044(what)N 780(fracton)X 1062(of)X 1168(the)X 1309(balls)X 1509(in)X 1611(the)X 1752(urn)X 1902(is)X 1994(white.)X 2245(The)X 2415(possible)X 2737(hypotheses)X 3161(state)X 3356(that)X 3522(some)X 3762(rational)X 576 2204(fraction)N 2 f 872(r)X 1 f 928(of)X 1023(balls)X 1212(in)X 1303(the)X 1433(urn)X 1572(is)X 1653(white,)X 1893(where)X 2129(0)X 9 f (\243)S 2 f 2221(r)X 1 f 9 f 2262(\243)X 1 f 2310(1.)X 2398(By)X 2524(the)X 2655(above)X 2888(rule,)X 3070(if)X 3147(in)X 2 f 3239(n)X 1 f 3306(draws,)X 2 f 3560(m)X 1 f 3646(white)X 3865(balls)X 576 2364(are)N 718(selected,)X 1059(then)X 1246(we)X 1383(should)X 1652(formulate)X 2029(the)X 2171(hypothesis)X 2 f 2582(r)X 9 f 2630(=)X 2 f 2678(m)X 2748(/)X 2780(n)X 1 f (.)S 2902(Let)X 3054(there)X 3264(be)X 3381(1/3)X 3528(white)X 3758(and)X 3919(2/3)X 576 2524(black)N 803(balls.)X 1028(However)X 1385(the)X 1529(probability)X 1953(of)X 2062(getting)X 2345(the)X 2490(true)X 2664(hypothesis)X 2 f 3078(m)X 9 f 3155(=)X 2 f 3203(n)X 3254(/)X 1 f 3286(3)X 3367(is)X 3463(zero)X 3651(if)X 2 f 3742(n)X 1 f 3823(is)X 3919(not)X 576 2684(divisible)N 913(by)X 1033(3,)X 1131(and)X 1290(it)X 1372(tends)X 1590(to)X 1691(zero,)X 1896(even)X 2094(under)X 2326(the)X 2466(assumption)X 2900(that)X 2 f 3065(n)X 1 f 3141(is)X 3232(divisible)X 3569(by)X 3689(3.)X 3786(On)X 3924(the)X 576 2844(other)N 784(hand)X 982(we)X 1111(know)X 1333(that)X 1493(to)X 1589(obtain)X 1837(a)X 1903(hypothesis)X 2307(1)X 2 f 2358(/)X 1 f 2390(3)X 2 f 9 f (-)S 1 f 9 f 2482(e\243)X 2 f 2569(r)X 1 f 9 f 2610(\243)X 1 f 2658(1)X 2 f 2709(/)X 1 f 2741(3)X 2 f 9 f (+)S 1 f 9 f 2833(e)X 1 f 2872(,)X 2921(for)X 3050(any)X 9 f 3204(e)X 1 f 3243(,)X 3292(has)X 3436(probability)X 3851(tend-)X 576 3004(ing)N 714(to)X 808(1)X 877(exponentially)X 1383(fast,)X 1557(by)X 1670(the)X 1803(so-called)X 2146(Chernoff)X 2487(formula.)X 2812(\(For)X 2987(Chernoff's)X 3391(formula)X 3694(see)X 3830(e.g.)X 4003([)X 576 3164(Angluin)N 887(Valiant)X 1169(\))X 1220(Even)X 1423(when)X 1635(the)X 1765(process)X 2050(converges,)X 2 f 2450(n)X 1 f 2516(may)X 2690(be)X 2795(too)X 2930(large)X 3128(for)X 3252(practical)X 3578(use.)X 3 f 776 3352(Kemeny's)N 1174(Rule.)X 2 f 1416(Select)X 1649(the)X 1779(simplest)X 2090(hypothesis)X 2484(compatible)X 2898(with)X 3073(the)X 3203(observed)X 3542(values.)X 1 f 576 3540(Here)N 772(`compatible')X 1249(is)X 1334(de\256ned)X 1619(as)X 1718(follows.)X 2052(The)X 2215(hypothesis)X 2 f 2618(H)X 8 s 2681 3557(i)N 1 f 11 s 2732 3540(is)N 2 f 2817(compatible)X 1 f 3235(with)X 3418(data)X 2 f 3591(D)X 1 f 3680(if,)X 3783(assum-)X 576 3700(ing)N 728(the)X 875(truth)X 1081(of)X 2 f 1193(H)X 8 s 1256 3717(i)N 1 f 11 s 1281 3700(,)N 1342(there)X 1557(was)X 1732(at)X 1835(most)X 2046(one)X 2212(percent)X 2510(chance)X 2793(of)X 2905(getting)X 3190(a)X 3267(deviation)X 3634(as)X 3745(great)X 3959(as)X 2 f 576 3860(m)N 1 f 653(\()X 2 f 682(H)X 8 s 745 3877(i)N 1 f 11 s 770 3860(,)N 2 f 799(D)X 1 f 869(\))X 931(for)X 1066(some)X 1285(measure)X 1611(function)X 2 f 1939(m)X 1 f 2002(.)X 2080(This)X 2271(is)X 2364(related)X 2638(to)X 2741(Valiant's)X 3098(learning)X 3421(theory)X 3680(to)X 3783(be)X 3900(dis-)X 576 4020(cussed)N 832(later.)X 776 4208(But)N 935(how)X 1117(does)X 1309(one)X 1467(de\256ne)X 1713(simplicit)X 2024(y?)X 2138(Is)X 2232(1)X 2 f 2283(/)X 1 f 2315(4)X 2390(simpler)X 2686(than)X 2869(1)X 2 f 2920(/)X 1 f 2952(10?)X 3132(Is)X 3226(1)X 2 f 3277(/)X 1 f 3309(3)X 3385(simpler)X 3682(than)X 3866(2)X 2 f 3917(/)X 1 f 3949(3?)X 576 4368(Saying)N 846(that)X 1004(an)X 1112(urn)X 1254(contains)X 1573(1)X 2 f 1624(/)X 1 f 1656(3rd)X 1798(part)X 1959(white)X 2179(balls)X 2370(comes)X 2619(down)X 2838(to)X 2931(the)X 3063(same)X 3268(thing)X 3474(as)X 3571(saying)X 3825(that)X 3982(it)X 576 4528(contains)N 901(a)X 971(2)X 2 f 1022(/)X 1 f 1054(3rd)X 1202(part)X 1370(black)X 1592(balls.)X 1812(Kemeny)X 2141(warned:)X 2455(`do)X 2603(not)X 2747(use)X 2895(more)X 3107(precision)X 3461(in)X 3561(your)X 3753(theories)X 576 4688(than)N 758(is)X 847(necessary.')X 1269(But)X 1427(what)X 1628(is)X 1717(necessary)X 2088(and)X 2245(what)X 2446(is)X 2535(not?)X 2739(All)X 2882(these)X 3092(issues)X 3331(are)X 3467(very)X 3652(subjective.)X 576 4848(Does)N 784(a)X 851(simple)X 1115(theory)X 1368(generate)X 1694(a)X 1761(hypothesis)X 2166(which)X 2409(is)X 2496(good)X 2700(for)X 2830(predicting)X 3217(future)X 3456(outcomes?)X 3862(How)X 576 5008(do)N 688(we)X 814(achieve)X 1106(fast)X 1256(convergence?)X 1784(How)X 1977(does)X 2161(one)X 2311(trade)X 2510(between)X 2826(`simplicity')X 3262(and)X 3412(`truth')X 3660(\(`compati-)X 576 5168(bility'\)?)N 906(Kemeny)X 1227(actually)X 1530(asked)X 1753(for)X 1878(`a)X 1969(criterion)X 2291(combining)X 2692(an)X 2798(optimum)X 3141(of)X 3238(simplicit)X 3549(y)X 3617(and)X 3768(compa-)X 576 5328(tibilit)N 770(y')X 865([crediting)X 1230(Nelson)X 1501(Goodman)X 1870(for)X 1994(this)X 2144(suggestion].)X ep %%Page: 7 7 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2231 424(-)N 2282(7)X 2348(-)X 3 f 576 736(1.1.)N 752(Combining)X 1195(Epicurus,)X 1582(Ockham,)X 1949(and)X 2113(Bayes)X 1 f 576 924(The)N 735(study)X 948(of)X 1044(inductive)X 1396(reasoning)X 1761(predates)X 2077(arti\256cial)X 2395(intelligence)X 2831(or)X 2927(computer)X 3283(science)X 3565(by)X 3676(more)X 3880(than)X 576 1084(2000)N 785(years.)X 1047(There)X 1285(is)X 1377(tremendous)X 1821(amount)X 2119(of)X 2224(literature)X 2575(in)X 2676(many)X 2904(different)X 3239(\256elds)X 3462(under)X 3694(diverging)X 576 1244(terminologies.)N 1130(Our)X 1312(goal)X 1510(is)X 1615(to)X 1730(extract)X 2016(a)X 2101(common)X 2457(core)X 2655(of)X 2775(simple)X 3058(ideas)X 3286(underlying)X 3715(all)X 3851(these)X 576 1404(approaches,)N 1033(in)X 1141(the)X 1288(spirit)X 1509(of)X 1621(Occam's)X 1972(Razor)X 2221(principle.)X 2618(We)X 2779(will)X 2956(start)X 3147(with)X 3343(Bayesian)X 3705(inference)X 576 1564(theory.)N 776 1752(To)N 897(apply)X 1116(Bayesian)X 1462(type)X 1637(reasoning)X 2002(one)X 2152(has)X 2292(to)X 2384(assign)X 2 f 2627(a)X 2695(priori)X 1 f 2925(probabilities)X 3391(\(prior)X 3615(probability\))X 576 1912(to)N 670(each)X 856(possible)X 1170(hypothesis.)X 1616(Since)X 1837(the)X 1969(posthumous)X 2419(publication)X 2841(in)X 2934(1763)X 3134(of)X 3231(Th.)X 3375(Bayes')X 3643(\(??)X 3774(-)X 3827(1761\))X 576 2072(famous)N 866(memoir)X 1172(`An)X 1340(essay)X 1562(towards)X 1872(solving)X 2164(a)X 2235(problem)X 2561(in)X 2662(the)X 2802(doctrine)X 3123(of)X 3228(chances')X 3567(by)X 3687(his)X 3822(friend)X 576 2232(Richard)N 888(Price,)X 1124([)X 1185(Bayes)X 1432(essay)X 1654(there)X 1862(has)X 2011(been)X 2209(continuous)X 2628(bitter)X 2847(debate)X 3109(on)X 3229(the)X 3369(controversial)X 3861(prior)X 576 2392(probability)N 986(in)X 1077(the)X 1207(Bayesian)X 1552(formula.)X 776 2580(The)N 945(invention)X 1311(of)X 1416(Kolmogorov)X 1898(complexity,)X 2350(by)X 2470(its)X 2587(\256rst)X 2757(inventor)X 3084(R.)X 3198(Solomonoff,)X 3674(was)X 3843(as)X 3949(an)X 576 2740(auxiliary)N 926(notion)X 1188(to)X 1293(resolve)X 1582(this)X 1745(particular)X 2118(problem.)X 2469(Namely,)X 2805(using)X 3031(Kolmogorov)X 3516(complexity)X 3949(he)X 576 2900(found)N 811(a)X 880(single)X 1121(`universal')X 1532(prior)X 1733(distribution)X 2171(which)X 2416(can)X 2568(be)X 2681(substituted)X 3094(for)X 3226(any)X 3383(particular)X 3752(actually)X 576 3060(valid)N 785(distribution)X 1225(\(as)X 1359(long)X 1548(as)X 1653(it)X 1735(is)X 1826(computable\))X 2299(in)X 2400(Bayes')X 2676(formula,)X 3008(and)X 3166(obtain)X 3418(approximately)X 3959(as)X 576 3220(good)N 798(results)X 1074(as)X 1193(if)X 1293(the)X 1447(actually)X 1773(valid)X 1996(distribution)X 2450(had)X 2623(been)X 2835(used.)X 3064(It)X 3164(sounds)X 3454(like)X 3634(magic,)X 3919(but)X 576 3380(Solomonoff's)N 1082(approach)X 1426(does)X 1609(give)X 1783(a)X 1844(more)X 2047(or)X 2142(less)X 2296(satisfactory)X 2724(solution)X 3031(to)X 3122(this)X 3272(unlikely)X 3584(objective.)X 776 3568(The)N 935(elegant)X 1212(idea)X 1381(of)X 1476(a)X 1538(universal)X 1884(prior)X 2078(is)X 2160(a)X 2222(combination)X 2687(of)X 2783(Occam's)X 3118(razor)X 3321(and)X 3471(modern)X 3763(compu-)X 576 3728(tability)N 851(theory.)X 1121(However,)X 1487(the)X 1618(universal)X 1964(prior)X 2158(is)X 2239(uncomputable,)X 2783(since)X 2986(it)X 3058(involves)X 3379(Kolmogorov)X 3851(com-)X 576 3888(plexity.)N 890(In)X 987(this)X 1139(paper)X 1358(we)X 1485(develop)X 1789(the)X 1922(thesis)X 2148(that)X 2306(many)X 2527(theories,)X 2853(models,)X 3155(and)X 3307(principles)X 3680(for)X 3807(induc-)X 576 4048(tive)N 758(reasoning)X 1149(that)X 1331(were)X 1550(formulated)X 1986(both)X 2192(before)X 2464(and)X 2639(after)X 2848(Solomonoff's)X 3380(inception,)X 3779(can)X 3949(be)X 576 4208(rigorously)N 960(derived)X 1246(as)X 1341(particular)X 1701(computable)X 2135(approximations)X 2706(to)X 2797(it.)X 776 4396(We)N 926(\256rst)X 1091(describe)X 1412(the)X 1548(basics)X 1791(of)X 1892(Bayesian)X 2243(theory)X 2496(and)X 2652(how)X 2832(to)X 2930(apply)X 3155(Kolmogorov)X 3634(complexity)X 576 4556(to)N 674(obtain)X 924(the)X 1061(Universal)X 1432(prior)X 1632(probability)X 2049(distribution.)X 2508(We)X 2659(then)X 2840(derive)X 3089(the)X 3226(Gold)X 3431(paradigm)X 3793(and)X 3948(its)X 576 4716(principles.)N 969(We)X 1114(derive)X 1357(a)X 1420(form)X 1615(of)X 1712(Rissanen's)X 2117(Minimum)X 2495(Description)X 2930(Length)X 3204(\(MDL\))X 3481(principle.)X 3841(From)X 576 4876(the)N 725(MDL)X 961(principle)X 1316(Rissanen)X 1675(derives)X 1970(the)X 2118(Fisher's)X 2441(Maximum)X 2849(Likelihood)X 3277(principle)X 3631(and)X 3798(Jaynes)X 576 5036(Maximum)N 968(Entropy)X 1277(principle.)X 1638(This)X 1820(paper)X 2040(contains)X 2359(a)X 2423(review)X 2687(of)X 2785(all)X 2899(these)X 3105(theories)X 3409(and)X 3561(principles.)X 3978(It)X 576 5196(has)N 729(been)X 931(our)X 1084(experience)X 1501(that)X 1670(some)X 1892(experts)X 2182(say)X 2335(the)X 2479(connections)X 2935(as)X 3043(claimed)X 3358(are)X 3500(obvious,)X 3836(while)X 576 5356(some)N 792(other)X 1003(experts)X 1287(deny)X 1488(those)X 1704(connections)X 2155(exist.)X 2374(Thus,)X 2602(since)X 2813(the)X 2951(proof)X 3171(of)X 3275(the)X 3414(pudding)X 3734(is)X 3824(in)X 3924(the)X 576 5516(eating,)N 850(we)X 988(explicitly)X 1359(establish)X 1704(derivations)X 2132(together)X 2457(with)X 2650(the)X 2794(appropriate)X 3231(related)X 3507(theorems.)X 3910(We)X 576 5676(also)N 745(describe)X 1065(an)X 1175(experiment)X 1599(we)X 1729(have)X 1923(performed)X 2317(in)X 2414(machine)X 2741(learning)X 3058(of)X 3159(recognition)X 3589(of)X 3690(handwrit-)X 576 5836(ten)N 718(characters)X 1107(using)X 1331(the)X 1472(MDL)X 1700(principle.)X 2091(Combination)X 2586(of)X 2692(Gold-style)X 3097(inference)X 3457(with)X 3647(ideas)X 3861(from)X ep %%Page: 8 8 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2231 424(-)N 2282(8)X 2348(-)X 576 736(computational)N 1119(complexity)X 1554(theory)X 1816(leads)X 2034(to)X 2140(Valiant's)X 2500(model)X 2758(of)X 2869(deductive)X 3250(learning.)X 3599(We)X 3759(give)X 3949(an)X 576 896(application)N 1008(of)X 1120(the)X 1267(Universal)X 1648(prior)X 1858(distribution)X 2305(to)X 2413(obtain)X 2672(a)X 2749(theory)X 3012(of)X 3123(learning)X 3450(simple)X 3724(concepts)X 576 1056(under)N 815(simple)X 1090(distributions.)X 1593(A)X 1695(more)X 1915(extensive)X 2287(treatment)X 2660(of)X 2772(this)X 2939(material)X 3269(will)X 3447(be)X 3570(given)X 3806(in)X 3915(our)X 576 1216(forth-coming)N 1063(textbook)X 1394([)X 1445(Li)X 1546(Introduction)X 2004(Kolmogorov)X 3 f 576 1536(2.)N 686(The)X 855(Universal)X 1239(Prior)X 1462(Distribution)X 576 1856(2.1.)N 752(Bayesian)X 1112(Inference)X 1 f 576 2044(In)N 680(the)X 819(following)X 1193(discussion)X 1591(of)X 1696(probability)X 2116(we)X 2250(assume)X 2541(the)X 2681(usual)X 2899(so-called)X 3249(Kolmogorov)X 3731(Axioms,)X 576 2204(see)N 725(e.g.)X 911([)X 977(Feller)X 1220(Introduction)X 1693(For)X 1851(our)X 2004(purpose)X 2318(we)X 2456(need)X 2658(the)X 2802(following.)X 3225(We)X 3383(have)X 3585(a)X 2 f 3660(hypothesis)X 576 2364(space)N 1 f 776(,)X 3 f 836(H)X 2 f 9 f 919(=)X 2 f 981({H)X 1 f 8 s 1089 2381(1)N 11 s 1128 2364(,)N 2 f 1171(H)X 1 f 8 s 1244 2381(2)N 11 s 1283 2364(,)N 2 f 1312(...})X 1 f 1413(,)X 1473(which)X 1726(consists)X 2044(of)X 2156(a)X 2234(countable)X 2616(set)X 2753(of)X 2865(hypotheses,)X 3317(which)X 3571(are)X 3717(mutually)X 576 2524(exclusive)N 933(\(in)X 1055(the)X 1187(sense)X 1401(that)X 1558(at)X 1646(most)X 1842(one)X 1993(of)X 2089(them)X 2289(is)X 2371(right\),)X 2612(and)X 2762(exhaustive)X 3162(\(in)X 3283(the)X 3414(sense)X 3627(that)X 3783(at)X 3870(least)X 576 2684(one)N 744(of)X 858(them)X 1076(is)X 1176(right\).)X 1457(With)X 1675(each)X 1877(hypothesis)X 2 f 2295(H)X 8 s 2358 2701(i)N 1 f 11 s 2424 2684(we)N 2567(associate)X 2926(a)X 2 f 3006(probability)X 3440(P)X 1 f 3508(\()X 2 f 3537(H)X 8 s 3600 2701(i)N 1 f 11 s 3625 2684(\))N 3696(such)X 3899(that)X 16 s 9 f 576 2870(S)N 2 f 8 s 652 2888(i)N 11 s 677 2844(P)N 1 f 745(\()X 2 f 774(H)X 8 s 837 2861(i)N 1 f 11 s 862 2844(\))N 2 f 9 f 905(=)X 1 f 967(1.)X 1096(The)X 1273(student)X 1568(is)X 1667(supplied)X 2006(with)X 2203(some)X 2429(data)X 2 f 2616(D)X 1 f 2679(,)X 2741(providing)X 3124(information)X 3581(about)X 3817(which)X 576 3030(hypothesis)N 1073(is)X 1252(correct.)X 1660(From)X 1971(the)X 2199(de\256nition)X 2658(of)X 2852(conditional)X 3371(probability,)X 3902(i.e.,)X 2 f 576 3190(P)N 1 f 644(\()X 2 f 673(A)X 1 f 9 f 755(|)X 2 f 801(B)X 1 f 862(\))X 2 f 9 f 905(=)X 2 f 967(P)X 1 f 1035(\()X 2 f 1064(A)X 9 f 1139(\307)X 2 f 1221(B)X 1 f 1282(\))X 2 f 1318(/)X 1350(P)X 1 f 1418(\()X 2 f 1447(B)X 1 f 1508(\),)X 1583(it)X 1657(is)X 1740(easy)X 1920(to)X 2013(derive)X 3 f 2257(Bayes')X 2530(formula)X 1 f 2856(\(rewrite)X 2 f 3157(P)X 1 f 3225(\()X 2 f 3254(A)X 9 f 3322(\307)X 2 f 3390(B)X 1 f 3451(\))X 3503(in)X 3595(the)X 3726(two)X 3881(pos-)X 576 3350(sible)N 765(different)X 1090(ways,)X 1314(equate)X 1566(the)X 1696(two)X 1850(expressions,)X 2304(and)X 2453(set)X 2 f 2573(A)X 9 f 2648(=)X 2 f 2710(H)X 8 s 2773 3367(i)N 1 f 11 s 2820 3350(and)N 2 f 2969(B)X 9 f 3044(=)X 2 f 3106(D)X 1 f 3169(\):)X 2 f 1782 3642(P)N 1 f 1850(\()X 2 f 1879(H)X 8 s 1942 3659(i)N 1 f 11 s 9 f 1981 3642(|)N 2 f 2027(D)X 1 f 2097(\))X 2 f 9 f 2148(=)X 1 f 2 f 2424 3703(P)N 1 f 2492(\()X 2 f 2521(D)X 1 f 2591(\))X 2 f 2240 3572(P)N 1 f 2308(\()X 2 f 2337(D)X 1 f 9 f 2428(|)X 2 f 2460(H)X 8 s 2523 3589(i)N 1 f 11 s 2548 3572(\))N 2 f 2591(P)X 1 f 2659(\()X 2 f 2688(H)X 8 s 2751 3589(i)N 1 f 11 s 2776 3572(\))N 10 f 2226 3616(h)N 2246(hhhhhhhhhhhhh)X 1 f 3930 3642(\(1\))N 576 3954(where)N 2 f 1835 4194(P)N 1 f 1903(\()X 2 f 1932(D)X 1 f 2002(\))X 2 f 9 f 2053(=)X 1 f 2 f 8 s 2152 4284(i)N 1 f 16 s 9 f 2123 4220(S)N 2 f 11 s 2199 4194(P)N 1 f 2267(\()X 2 f 2296(D)X 1 f 9 f 2387(|)X 2 f 2419(H)X 8 s 2482 4211(i)N 1 f 11 s 2507 4194(\))N 2 f 2536(P)X 1 f 2604(\()X 2 f 2633(H)X 8 s 2696 4211(i)N 1 f 11 s 2721 4194(\).)N 576 4534(We)N 732(interpret)X 1065(the)X 1207(different)X 1544(variables)X 1896(in)X 1999(the)X 2142(formula)X 2456(as)X 2564(follows.)X 2885(The)X 2 f 3057(H)X 8 s 3120 4551(i)N 1 f 11 s 3145 4534('s)N 3243(represent)X 3600(the)X 3743(possible)X 576 4694(alternative)N 980(hypotheses)X 1402(concerning)X 1824(the)X 1962(phenomenon)X 2447(we)X 2579(wish)X 2775(to)X 2874(discover.)X 3224(The)X 3391(term)X 2 f 3583(D)X 1 f 3676(represents)X 576 4854(the)N 711(empirically)X 1141(or)X 1241(otherwise)X 1610(known)X 1876(data)X 2050(concerning)X 2468(this)X 2623(phenomenon.)X 3149(The)X 3314(term)X 2 f 3504(P)X 1 f 3572(\()X 2 f 3601(D)X 1 f 3671(\),)X 3750(the)X 3886(pro-)X 576 5014(bability)N 877(of)X 980(data)X 2 f 1157(D)X 1 f 1220(,)X 1272(can)X 1424(be)X 1537(considered)X 1948(as)X 2051(a)X 2120(normalizing)X 2577(factor)X 2811(so)X 2918(that)X 16 s 9 f 3080 5040(S)N 2 f 8 s 3156 5058(i)N 11 s 3181 5014(P)N 1 f 3249(\()X 2 f 3278(H)X 8 s 3341 5031(i)N 1 f 11 s 9 f 3380 5014(|)N 2 f 3412(D)X 1 f 3482(\))X 2 f 9 f 3525(=)X 1 f 3587(1.)X 3704(The)X 3870(term)X 2 f 576 5200(P)N 1 f 644(\()X 2 f 673(H)X 8 s 736 5217(i)N 1 f 11 s 761 5200(\))N 821(is)X 911(called)X 1153(the)X 2 f 1292(prior)X 1 f 1504(probability)X 1923(or)X 2 f 2027(initial)X 1 f 2271(probability)X 2690(of)X 2794(hypothesis)X 2 f 3202(H)X 8 s 3265 5217(i)N 1 f 11 s 3290 5200(,)N 3343(i.e.,)X 3504(the)X 3644(probability)X 576 5360(that)N 2 f 754(H)X 8 s 817 5377(i)N 1 f 11 s 887 5360(is)N 991(true)X 1173(before)X 1442(we)X 1589(have)X 1800(seen)X 2001(any)X 2173(evidence.)X 2575(The)X 2756(term)X 2 f 2962(P)X 1 f 3030(\()X 2 f 3059(H)X 8 s 3122 5377(i)N 1 f 11 s 9 f 3161 5360(|)N 2 f 3193(D)X 1 f 3263(\))X 3336(is)X 3439(called)X 3694(the)X 2 f 3853(\256nal)X 1 f 4010(,)X 2 f 576 5520(a)N 1 f 2 f 649(posteriori)X 1 f 997(,)X 1046(or)X 2 f 1146(inferred)X 1 f 1457(probability,)X 1894(which)X 2136(re\257ects)X 2417(the)X 2552(probability)X 2967(of)X 2 f 3068(H)X 8 s 3131 5537(i)N 1 f 11 s 3184 5520(modi\256ed)N 3526(from)X 3725(the)X 3861(prior)X 576 5680(probability)N 2 f 988(P)X 1 f 1056(\()X 2 f 1085(H)X 8 s 1148 5697(i)N 1 f 11 s 1173 5680(\))N 1226(after)X 1411(seeing)X 1660(the)X 1791(data)X 2 f 1961(D)X 1 f 2024(.)X 2091(The)X 2251(term)X 2 f 2436(P)X 1 f 2504(\()X 2 f 2533(D)X 1 f 9 f 2624(|)X 2 f 2670(H)X 8 s 2733 5697(i)N 1 f 11 s 2758 5680(\),)N 2832(the)X 2963(conditional)X 3384(probability)X 3795(of)X 3891(see-)X 576 5840(ing)N 2 f 715(D)X 1 f 804(when)X 2 f 1020(H)X 8 s 1083 5857(i)N 1 f 11 s 1134 5840(is)N 1219(true,)X 1404(is)X 1489(assumed)X 1818(to)X 1913(be)X 2022(computable)X 2460(from)X 2 f 2657(D)X 1 f 2746(and)X 2 f 2899(H)X 8 s 2962 5857(i)N 1 f 11 s 2987 5840(.)N 3035(In)X 3134(many)X 3356(learning)X 3671(situations,)X ep %%Page: 9 9 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2231 424(-)N 2282(9)X 2348(-)X 576 736(data)N 757(can)X 913(only)X 1104(be)X 1220(consistent)X 1606(with)X 1796(an)X 1912(hypothesis)X 2 f 2322(H)X 8 s 2385 753(i)N 1 f 11 s 2443 736(in)N 2545(the)X 2686(sense)X 2909(of)X 3015(being)X 3244(forced)X 3501(by)X 3622(it)X 3705(such)X 3899(that)X 2 f 576 896(P)N 1 f 644(\()X 2 f 673(D)X 1 f 9 f 764(|)X 2 f 810(H)X 8 s 873 913(i)N 1 f 11 s 898 896(\))N 2 f 9 f 941(=)X 1 f 1003(1.)X 1119(If)X 1205(the)X 1341(data)X 1517(is)X 1605(inconsistent)X 2056(with)X 2242(hypothesis)X 2 f 2648(H)X 8 s 2711 913(i)N 1 f 11 s 2765 896(then)N 2 f 2946(P)X 1 f 3014(\()X 2 f 3043(D)X 1 f 9 f 3134(|)X 2 f 3166(H)X 8 s 3229 913(i)N 1 f 11 s 3254 896(\))N 2 f 9 f 3297(=)X 1 f 3359(0.)X 3476(In)X 3578(such)X 3768(a)X 3836(situa-)X 576 1056(tion,)N 771(the)X 914(data)X 1096(either)X 1332(is)X 1426(determined)X 1858(by)X 1981(a)X 2055(hypothesis,)X 2489(or)X 2597(disquali\256es)X 3034(it.)X 3163(\(We)X 3348(assume)X 3641(there)X 3851(is)X 3944(no)X 576 1216(noise)N 784(that)X 939(distorts)X 1221(the)X 1351(data.\))X 1593(For)X 1737(example,)X 2080(the)X 2210(hypothesis)X 2609(is)X 2690(datum)X 2 f 2933(x)X 9 f 2993(\316)X 2 f 3070(L)X 1 f 3141(or)X 2 f 3236(x)X 3310(L)X 1 f 3359(.)X 776 1404(The)N 942(most)X 1143(interesting)X 1545(term)X 1736(is)X 1824(the)X 1961(prior)X 2161(probability)X 2 f 2578(P)X 1 f 2646(\()X 2 f 2675(H)X 8 s 2738 1421(i)N 1 f 11 s 2763 1404(\).)N 2865(In)X 2967(the)X 3105(context)X 3395(of)X 3498(machine)X 3827(learn-)X 576 1564(ing,)N 2 f 734(P)X 1 f 802(\()X 2 f 831(H)X 8 s 894 1581(i)N 1 f 11 s 919 1564(\))N 971(is)X 1053(often)X 1257(considered)X 1661(as)X 1757(the)X 1888(learner's)X 2 f 2218(initial)X 2454(degree)X 2715(of)X 2806(belief)X 1 f 3025(in)X 3116(hypothesis)X 2 f 3515(H)X 8 s 3578 1581(i)N 1 f 11 s 3603 1564(.)N 3669(In)X 3764(essence)X 576 1724(Bayes')N 851(rule)X 1019(is)X 1109(a)X 2 f 1180(mapping)X 1 f 1520(from)X 2 f 1723(a)X 1799(priori)X 1 f 2037(probability)X 2 f 2457(P)X 1 f 2525(\()X 2 f 2554(H)X 8 s 2617 1741(i)N 1 f 11 s 2642 1724(\))N 2703(to)X 2 f 2804(a)X 2880(posteriori)X 1 f 3260(probability)X 2 f 3680(P)X 1 f 3748(\()X 2 f 3777(H)X 8 s 3840 1741(i)N 1 f 11 s 9 f 3879 1724(|)N 2 f 3911(D)X 1 f 3981(\),)X 576 1884(where)N 819(the)X 956(mapping)X 1294(is)X 1382(determined)X 1808(by)X 1925(data)X 2 f 2101(D)X 1 f 2164(.)X 2237(In)X 2339(general,)X 2649(the)X 2786(problem)X 3108(is)X 3195(not)X 3336(so)X 3442(much)X 3666(that)X 3827(in)X 3924(the)X 576 2044(limit)N 771(the)X 905(inferred)X 1209(probability)X 1623(would)X 1869(not)X 2009(`condense')X 2421(on)X 2536(the)X 2671(`true')X 2893(hypothesis,)X 3319(but)X 3459(that)X 3619(the)X 3754(inferred)X 576 2204(probability)N 988(gives)X 1198(as)X 1295(much)X 1515(information)X 1956(as)X 2053(possible)X 2366(about)X 2586(the)X 2718(possible)X 3031(hypotheses)X 3446(from)X 3641(only)X 3822(a)X 3884(lim-)X 576 2364(ited)N 736(number)X 1032(of)X 1132(data,)X 1328(cf.)X 1445(example)X 1771(below.)X 2058(In)X 2159(fact,)X 2341(the)X 2477(continuous)X 2892(bitter)X 3107(debate)X 3365(between)X 3686(the)X 3822(Baye-)X 576 2524(sian)N 746(and)X 901(non-Bayesian)X 1413(opinions)X 1744(centered)X 2069(on)X 2184(the)X 2 f 2319(prior)X 1 f 2527(probability.)X 2986(The)X 3150(controversy)X 3592(is)X 3678(caused)X 3944(by)X 576 2684(the)N 715(fact)X 878(that)X 1042(Bayesian)X 1396(theory)X 1652(does)X 1844(not)X 1988(say)X 2136(how)X 2318(to)X 2418(initiall)X 2651(y)X 2726(derive)X 2977(the)X 3116(prior)X 3318(probabilities)X 3791(for)X 3924(the)X 576 2844(hypotheses.)N 1038(Rather,)X 1322(Bayes')X 1593(rule)X 1757(only)X 1941(says)X 2119(how)X 2297(they)X 2476(are)X 2610(to)X 2706(be)X 2 f 2816(updated)X 1 f 3100(.)X 3170(However,)X 3539(in)X 3634(each)X 3821(actual)X 576 3004(case)N 753(the)X 887(prior)X 1084(probabilities)X 1552(may)X 1731(be)X 1841(unknown,)X 2217(uncomputable,)X 2766(or)X 2866(conceivably)X 3319(do)X 3434(not)X 3574(exist.)X 3812(\(What)X 576 3164(is)N 664(the)X 801(prior)X 1001(probability)X 1418(of)X 1520(use)X 1666(of)X 1768(words)X 2011(in)X 2108(written)X 2386(English?)X 2745(There)X 2978(are)X 3113(many)X 3337(different)X 3668(sources)X 3959(of)X 576 3324(different)N 901(social)X 1129(backgrounds)X 1600(living)X 1829(in)X 1921(different)X 2247(ages.\))X 2499(This)X 2679(problem)X 2996(is)X 3078(solved)X 3331(if)X 3408(we)X 3533(can)X 3678(\256nd)X 3838(a)X 2 f 3900(sin-)X 576 3484(gle)N 1 f 716(probability)X 1136(distribution)X 1576(to)X 1677(use)X 1826(as)X 1931(the)X 2071(prior)X 2274(distribution)X 2714(in)X 2815(each)X 3008(different)X 3342(case,)X 3546(with)X 3734(approxi-)X 576 3644(mately)N 847(the)X 985(same)X 1196(result)X 1422(as)X 1525(if)X 1609(we)X 1741(had)X 1898(used)X 2089(the)X 2227(real)X 2389(distribution.)X 2850(Surprisingly,)X 3339(this)X 3498(turns)X 3705(out)X 3849(to)X 3949(be)X 576 3804(possible)N 887(up)X 997(to)X 1088(some)X 1296(mild)X 1481(restrictions.)X 3 f 776 3992(Example.)N 1 f 1154(We)X 1299(use)X 1439(an)X 1545(example)X 1867(of)X 1963(von)X 2118(Mises)X 2351([)X 2403(von)X 2558(mises)X 2782(probability)X 3193(truth)X 3383(Let)X 3524(an)X 3631(urn)X 3772(contain)X 576 4152(many)N 799(dice)X 973(each)X 1161(with)X 1345(different)X 1675(attributes.)X 2075(A)X 2165(die)X 2300(with)X 2484(attribute)X 2 f 2806(p)X 1 f 2877(has)X 3020(probability)X 2 f 3434(p)X 1 f 3504(showing)X 3828(6)X 3898(in)X 3993(a)X 576 4312(random)N 874(throw.)X 1130(For)X 1281(convenience,)X 1772(assume)X 2060(the)X 2198(attribute)X 2523(set)X 2 f 2651(A)X 1 f 2735(is)X 2824(\256nite,)X 3058(and)X 3215(the)X 3353(difference)X 3739(between)X 576 4472(each)N 762(pair)X 924(of)X 1022(attributes)X 1376(is)X 1460(greater)X 1729(than)X 1906(2)X 9 f (e)S 1 f 1989(.)X 2058(Randomly)X 2451(draw)X 2651(a)X 2715(die)X 2848(from)X 3044(the)X 3177(urn,)X 3341(our)X 3483(task)X 3650(is)X 3734(to)X 3827(deter-)X 576 4632(mine)N 790(its)X 911(attribute.)X 1287(We)X 1446(do)X 1572(this)X 1738(by)X 1864(experimenting.)X 2434(Throw)X 2706(the)X 2852(die)X 2 f 2998(n)X 1 f 3080(times)X 3310(independently.)X 3892(If)X 3988(6)X 576 4792(shows)N 820(up)X 2 f 933(m)X 1 f 1021(times,)X 1260(we)X 1386(choose)X 1654(the)X 1786(attribute)X 2105(that)X 2262(is)X 2345(nearest)X 2618(to)X 2 f 2711(m)X 2781(/)X 2813(n)X 1 f (.)S 2925(Let)X 2 f 3067(H)X 8 s 3130 4809(p)N 1 f 11 s 3193 4792(be)N 3300(the)X 3432(event)X 3647(of)X 3744(drawing)X 576 4952(a)N 643(die)X 780(with)X 966(attribute)X 2 f 1290(p)X 1 f 1363(from)X 1563(an)X 1675(urn.)X 1865(Let)X 2 f 2012(D)X 8 s 2075 4969(q)N 1 f 11 s 2143 4952(be)N 2255(the)X 2392(experimental)X 2882(data)X 3058(such)X 3248(that)X 2 f 3410(m)X 1 f 3502(successes)X 3867(\(6's\))X 9 s 10 f 576 5040(hhhhhhhhhhhhhhhh)N 1 f 576 5156(*)N 631(Properly)X 898(speaking,)X 1191(formula)X 1438(\(1\))X 1541(is)X 1608(not)X 1719(due)X 1842(to)X 1917(Bayes,)X 2130(but)X 2241(it)X 2300(is)X 2367(due)X 2490(to)X 2565(P.S.)X 2700(Laplace)X 2947(\(1749)X 3134(-)X 3177(1827\))X 3364(who)X 3508(stated)X 3696(the)X 3804(formula)X 576 5244(and)N 700(attached)X 960(Bayes')X 1180(name)X 1356(to)X 1432(it)X 1492([)X 1536(Laplace)X 1784(Actually,)X 2070(Bayes)X 2266(in)X 2342(his)X 2446(original)X 2690(paper)X 2870([)X 2914(Bayes)X 3110(Essay)X 3298(assumed)X 3566(the)X 3673(uniform)X 3924(dis-)X 576 5332(tribution)N 845(for)X 950(the)X 1059(a)X 1112(priori)X 1294(probability,)X 1650(hence)X 1840(he)X 1930(has)X 2048(essentially)X 2374(derived)X 2 f 2612(P)X 1 f 2668(\()X 2 f (H)S 6 s 5346(i)Y 1 f 9 s 9 f 2775 5332(|)N 2 f 2801(D)X 1 f 2859(\))X 2 f 9 f (=)S 2 f 2923(P)X 1 f 2979(\()X 2 f (D)S 1 f 9 f 3079(|)X 2 f 3105(H)X 6 s 5346(i)Y 1 f 9 s 3176 5332(\))N 2 f 3206(/)X 1 f 14 s 9 f 3232 5353(S)N 2 f 6 s 3298 5368(i)N 9 s 3317 5332(P)N 1 f 3373(\()X 2 f (D)S 1 f 9 f 3473(|)X 2 f 3499(H)X 6 s 5346(i)Y 1 f 9 s 3570 5332(\).)N 3634(Although)X 3928(this)X 576 5441(formula)N 827(can)X 950(be)X 1041(derived)X 1280(from)X 1443(\(1\))X 1550(by)X 1645(simply)X 1864(assuming)X 2159(that)X 2290(all)X 2 f 2385(P)X 1 f 2441(\()X 2 f (H)S 6 s 5455(i)Y 1 f 9 s 2536 5441(\))N 2583(are)X 2694(the)X 2805(same,)X 2994(Bayes)X 3193(did)X 3308(not)X 3423(state)X 3578(his)X 3684(result)X 3866(in)X 3944(the)X 576 5529(general)N 806(form)X 964(as)X 1042(in)X 1116(\(1\),)X 1236(nor)X 1350(did)X 1460(he)X 1546(derived)X 1780(his)X 1882(result)X 2060(through)X 2303(a)X 2354(formula)X 2601(similar)X 2820(to)X 2895(\(1\).)X 3034(Despite)X 3273(the)X 3380(fact)X 3507(that)X 3634(Bayes')X 3853(rule)X 3984(is)X 576 5617(just)N 699(a)X 750(rewriting)X 1033(of)X 1112(the)X 1219(de\256nition)X 1514(of)X 1593(conditional)X 1936(probability)X 2271(and)X 2394(nothing)X 2633(more,)X 2818(it)X 2877(is)X 2944(its)X 3030(interpretation)X 3436(and)X 3558(applications)X 3924(that)X 576 5705(are)N 682(most)X 840(profound)X 1122(and)X 1244(caused)X 1458(much)X 1636(bitter)X 1806(controversy)X 2164(during)X 2370(the)X 2476(past)X 2610(two)X 2736(centuries.)X ep %%Page: 10 10 bp 9 s 0 xH 0 xS 1 f 11 s 10 s 0 32(--)N 4323(--)X 11 s 2209 424(-)N 2260(10)X 2370(-)X 576 736(were)N 768(observed)X 1107(out)X 1242(of)X 2 f 1337(n)X 1 f 1403(throws)X 1664(and)X 9 f 1827(|)X 2 f 1859(q)X 9 f 1924(-)X 2 f 1986(m)X 2063(/)X 2095(n)X 1 f 9 f 2160(|)X 2 f 2206(<)X 1 f 9 f 2279(e)X 1 f 2318(,)X 2362(for)X 2 f 2486(q)X 9 f 2537(\316)X 2 f 2600(A)X 1 f 2654(.)X 2698(So)X 2 f 1725 1028(P)N 1 f 1793(\()X 2 f 1822(H)X 8 s 1885 1045(p)N 1 f 11 s 9 f 1938 1028(|)N 2 f 1970(D)X 8 s 2033 1045(q)N 1 f 11 s 2072 1028(\))N 2 f 9 f 2123(=)X 1 f 2 f 2410 1089(P)N 1 f 2478(\()X 2 f 2507(D)X 8 s 2570 1106(q)N 1 f 11 s 2609 1089(\))N 2 f 2215 958(P)N 1 f 2283(\()X 2 f 2312(D)X 8 s 2375 975(q)N 1 f 11 s 9 f 2428 958(|)N 2 f 2460(H)X 8 s 2523 975(p)N 1 f 11 s 2562 958(\))N 2 f 2605(P)X 1 f 2673(\()X 2 f 2702(H)X 8 s 2765 975(p)N 1 f 11 s 2804 958(\))N 10 f 2201 1002(h)N 2230(hhhhhhhhhhhhhh)X 1 f 2854 1028(,)N 576 1357(where)N 2 f 822(P)X 1 f 890(\()X 2 f 919(D)X 8 s 982 1374(q)N 1 f 11 s 1021 1357(\))N 2 f 9 f 1064(=)X 1 f 16 s 9 f 1126 1383(S)N 2 f 8 s 1202 1401(p)N 11 s 1241 1357(P)N 1 f 1309(\()X 2 f 1338(D)X 8 s 1401 1374(q)N 1 f 11 s 9 f 1454 1357(|)N 2 f 1500(H)X 8 s 1563 1374(p)N 1 f 11 s 1602 1357(\))N 2 f 1645(P)X 1 f 1713(\()X 2 f 1742(H)X 8 s 1805 1374(p)N 1 f 11 s 1844 1357(\).)N 1949(According)X 2352(to)X 2453(Chernoff's)X 2865(formula)X 3176(\(see)X 3349([)X 3410(angluin)X 3707(valiant)X 3981(\),)X 576 1543(for)N 9 f 700(a)X 2 f 755(<)X 1 f 814(1,)X 902(we)X 1026(have:)X 2 f 1731 1859(P)N 1 f 1799(\()X 2 f 1828(m)X 9 f 1905(-)X 2 f 1953(np)X 2048(>)X 1 f 9 f 2107(a)X 2 f 2162(np)X 1 f 9 f 2271(|)X 2 f 2317(H)X 8 s 2380 1876(p)N 1 f 11 s 2419 1859(\))N 2 f 2470(<)X 1 f 2 f 2551(e)X 8 s 9 f 2600 1779(-)N 1 f 2657 1823(2)N 2657 1741(1)N 10 f 2641 1760(hh)N 9 f 2711 1779(a)N 5 s 1 f 2751 1754(2)N 2 f 8 s 2776 1779(np)N 1 f 11 s 2847 1859(,)N 2 f 1735 2175(P)N 1 f 1803(\()X 2 f 1832(np)X 9 f 1934(-)X 2 f 1982(m)X 2052(>)X 1 f 9 f 2111(a)X 2 f 2166(np)X 1 f 9 f 2275(|)X 2 f 2321(H)X 8 s 2384 2192(p)N 1 f 11 s 2423 2175(\))N 2 f 2474(<)X 1 f 2 f 2555(e)X 8 s 9 f 2604 2095(-)N 1 f 2661 2139(3)N 2661 2057(1)N 10 f 2645 2076(hh)N 9 f 2715 2095(a)N 5 s 1 f 2755 2070(2)N 2 f 8 s 2780 2095(np)N 11 s 2851 2175(.)N 1 f 576 2415(Hence,)N 844(if)X 2 f 920(p)X 1 f 986(is)X 1067(the)X 1197(true)X 1356(attribute)X 1673(of)X 1768(the)X 1898(die)X 2028(we)X 2152(have)X 2340(drawn)X 2581(then,)X 2777(choosing)X 9 f 3117(a)X 2 f 9 f 3172(=)X 2 f 3220(p)X 3271(/)X 1 f 3303(2)X 9 f (e)S 1 f 3408(\(so)X 9 f 3552(|)X 2 f 3584(p)X 9 f 3649(-)X 2 f 3711(m)X 3781(/)X 3813(n)X 1 f 9 f 3885(|)X 3931(\263)X 3993(e)X 1 f 576 2575(implies)N 9 f 888(|)X 2 f 920(m)X 9 f 997(-)X 2 f 1045(np)X 1 f 9 f 1161(|)X 2 f 1207(>)X 1 f 9 f 1280(a)X 2 f 1335(np)X 1 f (\),)S 2 f 1511(P)X 1 f 1579(\()X 2 f 1608(D)X 8 s 1671 2592(q)N 1 f 11 s 9 f 1724 2575(|)N 2 f 1770(H)X 8 s 1833 2592(p)N 1 f 11 s 1872 2575(\))N 1937(goes)X 2134(to)X 2239(0)X 2319(exponentially)X 2836(fast)X 2999(when)X 3225(the)X 3369(number)X 3674(of)X 3783(experi-)X 576 2735(ments)N 811(increases,)X 1179(for)X 2 f 1305(q)X 9 f 1356(\271)X 2 f 1404(p)X 1 f (,)S 1494(and)X 2 f 1645(P)X 1 f 1713(\()X 2 f 1742(D)X 8 s 1805 2752(p)N 1 f 11 s 9 f 1858 2735(|)N 2 f 1904(H)X 8 s 1967 2752(p)N 1 f 11 s 2006 2735(\))N 2059(goes)X 2244(to)X 2338(1)X 2407(at)X 2496(the)X 2629(same)X 2835(rate.)X 3036(Hence)X 2 f 3285(P)X 1 f 3353(\()X 2 f 3382(H)X 8 s 3445 2752(p)N 1 f 11 s 9 f 3498 2735(|)N 2 f 3530(D)X 8 s 3593 2752(p)N 1 f 11 s 3632 2735(\))N 3686(goes)X 3872(to)X 3966(1.)X 576 2895(Thus)N 778(we)X 906(derive)X 1152(the)X 1286(correct)X 1556(attribute)X 1877(of)X 1976(the)X 2110(die)X 2244(with)X 2427(high)X 2610(probability)X 3024(\(using)X 3270(a)X 3335(polynomial)X 3763(number)X 576 3055(of)N 672(throws\).)X 985(The)X 1145(interesting)X 1541(point)X 1746(is)X 1828(that)X 1984(if)X 2061(the)X 2192(number)X 2484(of)X 2580(trials)X 2780(is)X 2862(small,)X 3099(then)X 3275(the)X 3407(inferred)X 3709(probabil-)X 576 3215(ity)N 2 f 693(P)X 1 f 761(\()X 2 f 790(H)X 8 s 853 3232(p)N 1 f 11 s 9 f 906 3215(|)N 2 f 938(D)X 8 s 1001 3232(q)N 1 f 11 s 1040 3215(\))N 1092(may)X 1267(heavily)X 1550(depend)X 1827(on)X 1937(the)X 2067(the)X 2197(prior)X 2390(probability)X 2 f 2800(P)X 1 f 2868(\()X 2 f 2897(H)X 8 s 2960 3232(p)N 1 f 11 s 2999 3215(\).)N 3072(However,)X 3437(if)X 2 f 3513(n)X 1 f 3579(is)X 3660(large,)X 3880(then)X 576 3375(irrespective)N 1019(of)X 1125(the)X 1266(prior)X 1470(distribution,)X 1933(the)X 2074(inferred)X 2385(probability)X 2806(condenses)X 3200(more)X 3414(and)X 3574(more)X 3788(around)X 2 f 576 3535(m)N 646(/)X 678(n)X 1 f (.)S 3 f 776 3723(Example.)N 1 f 1181(We)X 1331(explain)X 1619(a)X 1686(simpli\256ed)X 2068(version)X 2355(of)X 2456(Solomonoff's)X 2969(theory)X 3223(of)X 3325(inductive)X 3683(inference.)X 576 3883(The)N 752(simpli\256cati)X 1151(on)X 1278(is)X 1376(in)X 1484(that)X 1656(we,)X 1819(for)X 1960(the)X 2107(moment,)X 2458(consider)X 2794(only)X 2989(a)X 3066(discrete)X 3378(sample)X 3666(space)X 3899(like)X 2 f 576 4043({)N 1 f 611(0,)X 698(1)X 2 f (}*)S 1 f 821(,)X 871(The)X 1036(set)X 1162(of)X 1263(all)X 1380(\256nite)X 1590(binary)X 1843(sequences,)X 2249(rather)X 2482(than)X 2 f 2662({)X 1 f 2697(0,)X 2784(1)X 2 f (})S 1 f 8 s 9 f 2863 4008(\245)N 11 s 1 f 2916 4043(,)N 2966(the)X 3103(set)X 3230(of)X 3332(all)X 3450(one-way)X 3781(in\256nite)X 576 4203(binary)N 823(sequences.)X 776 4391(We)N 927(view)X 1127(theory)X 1381(formation)X 1758(in)X 1856(science)X 2144(as)X 2246(the)X 2383(process)X 2675(of)X 2777(obtaining)X 3140(a)X 3208(compact)X 3537(description)X 3959(of)X 576 4551(the)N 711(past)X 880(observations)X 1352(together)X 1668(with)X 1852(predictions)X 2271(of)X 2370(future)X 2606(ones.)X 2815(The)X 2978(investigator)X 3421(observes)X 3754(increas-)X 576 4711(ingly)N 784(larger)X 1015(initial)X 1250(segments)X 1605(of)X 1705(an)X 1815(\256nite)X 2024(binary)X 2276(sequence)X 2625(as)X 2725(the)X 2860(outcome)X 3191(of)X 3291(an)X 3401(\256nite)X 3610(sequence)X 3959(of)X 576 4871(experiments)N 1034(on)X 1149(some)X 1362(aspect)X 2 f 1609(X)X 1 f 1690(of)X 1790(nature.)X 2081(To)X 2205(describe)X 2524(the)X 2658(underlying)X 3066(regularity)X 3435(of)X 3534(this)X 3688(sequence,)X 576 5031(the)N 716(investigator)X 1165(tries)X 1349(to)X 1450(formulate)X 1825(a)X 1896(theory)X 2153(that)X 2318(governs)X 2 f 2628(X)X 1 f 2682(,)X 2736(on)X 2856(the)X 2996(basis)X 3204(of)X 3309(the)X 3449(outcome)X 3785(of)X 3890(past)X 576 5191(experiments.)N 1063(Candidate)X 1455(theories)X 1768(\(hypotheses\))X 2251(are)X 2392(identi\256ed)X 2760(with)X 2951(computer)X 3318(programs)X 3684(that)X 3851(com-)X 576 5351(pute)N 750(binary)X 997(sequences)X 1375(starting)X 1662(with)X 1841(the)X 1971(observed)X 2310(initial)X 2540(segment.)X 776 5539(First)N 963(assume)X 1247(the)X 1380(existence)X 1733(of)X 1831(a)X 1895(prior)X 2091(probability)X 2504(distribution,)X 2959(described)X 3321(by)X 3434(probability)X 3847(func-)X 576 5699(tion)N 2 f 786(P)X 1 f 840(,)X 934(over)X 1162(a)X 1273(discrete)X 1619(sample)X 1941(space)X 9 f 2208(W)X 2 f 9 f 2290(=)X 2 f 2352({)X 1 f 2387(0,)X 2460(1)X 2 f (}*)S 1 f 2583(.)X 2676(De\256ne)X 2981(a)X 3091(function)X 9 f 3456(m)X 1 f 3578(over)X 9 f 3805(W)X 1 f 3944(by)X 9 f 576 5859(m)N 1 f 627(\()X 2 f 656(x)X 1 f 702(\))X 2 f 9 f 745(=)X 1 f 16 s 9 f 807 5885(S)N 2 f 11 s 897 5859({P)N 1 f 1000(\()X 2 f 1029(xy)X 1 f 1114(\):)X 2 f 1182(y)X 9 f 1242(\316)X 1 f 9 f 1319(W)X 2 f 1387(})X 1 f 1422(.)X 1481(Thus,)X 9 f 1716(m)X 1 f 1767(\()X 2 f 1796(x)X 1 f 1842(\))X 1908(is)X 2004(the)X 2149(probability)X 2574(of)X 2684(a)X 2761(sequence)X 3121(starting)X 3424(with)X 2 f 3619(x)X 1 f 3658(.)X 3740(Given)X 3993(a)X ep %%Page: 11 11 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2209 424(-)N 2260(11)X 2370(-)X 576 736(previously)N 974(observed)X 1317(data)X 1490(string)X 2 f 1717(S)X 1 f (,)S 1809(the)X 1943(inference)X 2296(problem)X 2616(is)X 2701(to)X 2796(predict)X 3067(the)X 3201(next)X 3378(symbol)X 3663(in)X 3757(the)X 3890(out-)X 576 896(put)N 712(sequence,)X 1079(i.e.,)X 1232(extrapolating)X 1721(the)X 1852(sequence)X 2 f 2197(S)X 1 f (.)S 2286(In)X 2382(terms)X 2601(of)X 2697(the)X 2828(variables)X 3169(in)X 3261(Formula)X 3583(\(1\),)X 2 f 3730(H)X 8 s 3793 913(i)N 1 f 11 s 3841 896(is)N 3924(the)X 576 1056(hypothesis)N 979(that)X 1138(the)X 1272(sequence)X 1620(under)X 1846(consideration)X 2347(starts)X 2559(with)X 2742(initial)X 2976(segment)X 2 f 3296(S)X 3361(a)X 1 f (.)S 3453(The)X 3616(data)X 2 f 3788(D)X 1 f 3876(con-)X 576 1216(sist)N 724(in)X 823(the)X 961(assertion)X 1304(that)X 1467(the)X 1605(sequence)X 1957(in)X 2056(fact)X 2218(starts)X 2434(with)X 2622(initial)X 2861(segment)X 2 f 3186(S)X 1 f (.)S 3305(Thus,)X 3534(for)X 2 f 3667(P)X 1 f 3728(\()X 2 f 3757(H)X 8 s 3820 1233(i)N 1 f 11 s 3845 1216(\))N 3905(and)X 2 f 576 1376(P)N 1 f 644(\()X 2 f 673(D)X 1 f 743(\))X 794(in)X 885(Formula)X 1206(\(1\))X 1330(we)X 1454(substitute)X 9 f 1815(m)X 1 f 1866(\()X 2 f 1895(S)X 1960(a)X 1 f 2011(\))X 2062(and)X 9 f 2211(m)X 1 f 2262(\()X 2 f 2291(S)X 1 f 2342(\),)X 2415(respectively,)X 2885(and)X 3034(obtain,)X 2 f 3299(a)X 9 f 3364(=)X 1 f 3426(0)X 3492(or)X 2 f 3587(a)X 9 f 3652(=)X 1 f 3714(1,)X 2 f 1777 1651(P)N 1 f 1845(\()X 2 f 1874(Sa)X 1 f 9 f 1990(|)X 2 f 2036(S)X 1 f 2087(\))X 2 f 9 f 2138(=)X 1 f 9 f 2428 1712(m)N 1 f 2479(\()X 2 f 2508(S)X 1 f 2559(\))X 2 f 2230 1599(P)N 1 f 2298(\()X 2 f 2327(S)X 1 f 9 f 2399(|)X 2 f 2445(Sa)X 1 f 2540(\))X 9 f 2583(m)X 1 f 2634(\()X 2 f 2663(Sa)X 1 f 2758(\))X 10 f 2216 1625(h)N 2228(hhhhhhhhhhhhh)X 2 f 2808 1651(.)N 1 f 576 1935(Here)N 9 f 768(m)X 1 f 819(\()X 2 f 848(S)X 1 f 899(\))X 2 f 9 f 942(=)X 2 f 1004(P)X 1 f 1072(\()X 2 f 1101(S)X 1 f 9 f 1173(|)X 2 f 1219(S)X 1 f 1277(0\))X 9 f 1364(m)X 1 f 1415(\()X 2 f 1444(S)X 1 f 1502(0\))X 2 f 9 f 1589(+)X 2 f 1651(P)X 1 f 1719(\()X 2 f 1748(S)X 1 f 9 f 1820(|)X 2 f 1866(S)X 1 f 1924(1\))X 9 f 2011(m)X 1 f 2062(\()X 2 f 2091(S)X 1 f 2149(1\))X 2 f 9 f 2236(+)X 2 f 2298(P)X 1 f 2366(\()X 2 f 2395(S)X 1 f 2446(\).)X 2541(Obviously,)X 2 f 2952(P)X 1 f 3020(\()X 2 f 3049(S)X 1 f 9 f 3121(|)X 2 f 3167(Sa)X 1 f 3262(\))X 2 f 9 f 3305(=)X 1 f 3367(1)X 3433(for)X 3557(any)X 2 f 3706(a)X 1 f (,)S 3794(hence)X 2 f 1954 2210(P)N 1 f 2022(\()X 2 f 2051(Sa)X 1 f 9 f 2167(|)X 2 f 2213(S)X 1 f 2264(\))X 2 f 9 f 2315(=)X 1 f 9 f 2429 2271(m)N 1 f 2480(\()X 2 f 2509(S)X 1 f 2560(\))X 9 f 2407 2158(m)N 1 f 2458(\()X 2 f 2487(Sa)X 1 f 2582(\))X 10 f 2393 2184(h)N 2404(hhhhh)X 2 f 2632 2210(.)N 1 f 3930(\(2\))X 576 2494(In)N 676(terms)X 899(of)X 999(inductive)X 1355(inference)X 1709(or)X 1810(machine)X 2137(learning,)X 2476(the)X 2612(\256nal)X 2797(probability)X 2 f 3213(P)X 1 f 3281(\()X 2 f 3310(Sa)X 1 f 9 f 3426(|)X 2 f 3472(S)X 1 f 3523(\))X 3580(is)X 3667(the)X 3803(proba-)X 576 2654(bility)N 792(of)X 893(the)X 1029(next)X 1208(symbol)X 1495(being)X 2 f 1718(a)X 1 f (,)S 1811(given)X 2034(the)X 2169(initial)X 2404(sequence)X 2 f 2753(S)X 1 f (.)S 2846(Obviously)X 3240(we)X 3369(now)X 3547(only)X 3731(need)X 3924(the)X 576 2814(prior)N 769(probability)X 1179(to)X 1270(evaluate)X 2 f 1586(P)X 1 f 1654(\()X 2 f 1683(Sa)X 1 f 9 f 1799(|)X 2 f 1845(S)X 1 f 1896(\).)X 776 3002(The)N 937(goal)X 1113(of)X 1210(inductive)X 1563(inference)X 1915(in)X 2009(general)X 2293(is)X 2377(to)X 2471(be)X 2579(able)X 2751(to)X 2845(either)X 3071(\(i\))X 3179(predict)X 3449(\(extrapolate\))X 3924(the)X 576 3162(next)N 752(element)X 1056(of)X 2 f 1153(S)X 1 f (,)S 1243(or)X 1340(\(ii\))X 1472(to)X 1565(infer)X 1755(an)X 1862(underlying)X 2268(effective)X 2600(process)X 2887(\(in)X 3009(the)X 3141(most)X 3336(general)X 3618(case,)X 3814(a)X 3876(Tur-)X 576 3322(ing)N 722(machine,)X 1076(according)X 1456(to)X 1558(the)X 1699(Church-Turing)X 2260(thesis\))X 2523(that)X 2689(generated)X 2 f 3064(S)X 1 f (,)S 3163(and)X 3323(hence)X 3562(to)X 3665(be)X 3782(able)X 3963(to)X 576 3482(predict)N 843(the)X 973(next)X 1147(symbol.)X 776 3670(In)N 874(order)X 1084(to)X 1178(solve)X 1389(the)X 1523(problem)X 1843(for)X 1971(unknown)X 2324(prior)X 2521(probability,)X 2957(Solomonoff)X 3404(proposed)X 3752(what)X 3949(he)X 576 3830(called)N 818(a)X 3 f 887(universal)X 1265(prior)X 1491(distribution)X 1 f 1937(.)X 2011(We)X 2163(now)X 2344(carefully)X 2687(de\256ne)X 2932(the)X 3070(universal)X 3423(prior)X 3624(distribution)X 576 3990(and)N 728(prove)X 953(several)X 1227(fundamental)X 1693(theorems)X 2041(due)X 2193(to)X 2287(Solomonoff)X 2733(and)X 2886(L.A.)X 3073(Levin,)X 3327(and)X 3480(afterwards)X 3876(con-)X 576 4150(tinue)N 788(this)X 951(example.)X 1307(The)X 1478(de\256nitions)X 1885(and)X 2046(theorems)X 2403(are)X 2544(so)X 2656(fundamental)X 3131(that)X 3298(our)X 3449(approach)X 3805(totally)X 576 4310(rests)N 770(upon)X 979(them.)X 1211(These)X 1454(results)X 1717(are)X 1857(in)X 1959(some)X 2178(form)X 2382(hidden)X 2655(in)X 2757([)X 2820(Solomonoff)X 3275(1964)X 3485(])X 3548([)X 3611(Solomonoff)X 576 4470(IEEE)N 793(1978)X 995(convergence)X 1465(])X 1519([)X 1573(Levin)X 1804(Zvonkin)X 2128(])X 2182([)X 2236(G)X 2314(\302)X 2299(acs)X 2436(Lecture)X 2730(Notes)X 2960(1987)X 3161(For)X 3308(various)X 3592(reasons)X 3880(they)X 576 4630(are)N 709(dif\256cult)X 1015(to)X 1110(access,)X 1382(and)X 1535(almost)X 1797(unknown)X 2150(except)X 2406(to)X 2501(a)X 2566(few)X 2723(people)X 2985(doing)X 3213(research)X 3532(in)X 3628(this)X 3783(area.)X 3978(It)X 576 4790(seems)N 813(useful)X 1050(to)X 1141(recapitulate)X 1575(them.)X 1818(First)X 2002(we)X 2126(need)X 2314(the)X 2444(basic)X 2647(de\256nitions)X 3042(of)X 3137(Kolmogorov)X 3609(complexity.)X 3 f 576 5110(2.2.)N 752(Kolmogorov)X 1244(Complexity)X 1 f 576 5298(Inductive)N 936(reasoning)X 1305(was)X 1468(the)X 1603(midwife)X 1924(that)X 2084(stood)X 2302(at)X 2393(the)X 2528(cradle)X 2770(of)X 2870(Kolmogorov)X 3347(complexity.)X 3794(Nowa-)X 576 5458(days,)N 813(Kolmogorov)X 1317(complexity)X 1769(has)X 1940(been)X 2160(applied)X 2474(in)X 2597(many)X 2847(areas)X 3081(of)X 3207(computer)X 3593(science)X 3905(and)X 576 5618(mathematics)N 1048(\(see)X 1214([)X 1268(Kolmogorov)X 1743(complexity)X 2166(handbook)X 2539(for)X 2667(a)X 2732(general)X 3017(survey\),)X 3328(and)X 3481(few)X 3638(realize)X 3899(that)X 576 5778(Kolmogorov)N 1049(complexity)X 1470(was)X 1629(at)X 1716(\256rst)X 1876(invented)X 2203(for)X 2328(the)X 2458(purpose)X 2758(of)X 2853(inductive)X 3204(inference.)X 3575(In)X 3670(this)X 3820(essay,)X ep %%Page: 12 12 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2209 424(-)N 2260(12)X 2370(-)X 576 736(we)N 700(go)X 810(back)X 998(to)X 1089(this)X 1239(origin.)X 776 924(We)N 927(are)X 1063(interested)X 1435(in)X 1533(de\256ning)X 1851(the)X 1988(complexity)X 2415(of)X 2517(a)X 2585(concrete)X 2912(individual)X 3301(\256nite)X 3513(string)X 3744(of)X 3847(zeros)X 576 1084(and)N 738(ones.)X 956(Unless)X 1230(otherwise)X 1607(speci\256ed,)X 1977(all)X 2101(strings)X 2371(will)X 2544(be)X 2662(binary)X 2921(and)X 3082(of)X 3189(\256nite)X 3405(length.)X 3704(All)X 3851(loga-)X 576 1244(rithms)N 827(in)X 921(this)X 1074(paper)X 1294(are)X 1427(base)X 1609(2,)X 1701(unless)X 1947(it)X 2023(is)X 2108(explicitly)X 2469(noted)X 2691(they)X 2869(are)X 3002(not.)X 3163(If)X 2 f 3247(x)X 1 f 3312(is)X 3397(a)X 3462(string,)X 3711(then)X 2 f 3889(l)X 1 f 3928(\()X 2 f 3957(x)X 1 f 4003(\))X 576 1404(denotes)N 872(the)X 2 f 1007(length)X 1 f 1255(\(number)X 1580(of)X 1680(zeros)X 1892(and)X 2045(ones\))X 2261(of)X 2 f 2360(x)X 1 f 2399(.)X 2469(We)X 2617(identify)X 2918(throughout)X 3331(the)X 2 f 3465(x)X 1 f 3504(th)X 3599(\256nite)X 3807(binary)X 576 1564(string)N 799(with)X 978(the)X 1108(natural)X 1375(number)X 2 f 1666(x)X 1 f 1705(,)X 1749(according)X 2118(to)X 2209(the)X 2339(correspondence:)X 1518 1804(\()N 9 f 1547(e)X 1 f 1586(,)X 1629(0\),)X 1745(\(0,)X 1861(1\),)X 1977(\(1,)X 2093(2\),)X 2209(\(00,)X 2369(3\),)X 2485(\(01,)X 2645(4\),)X 2761(\(10,)X 2921(5\),)X 3023(...)X 576 2044(Intuitively,)N 989(we)X 1113(want)X 1306(to)X 1397(call)X 1547(a)X 1608(string)X 1831(simple)X 2089(if)X 2165(it)X 2237(can)X 2381(be)X 2486(described)X 2845(in)X 2937(a)X 2999(few)X 3153(words,)X 3412(like)X 3568("the)X 3735(string)X 3959(of)X 576 2204(a)N 646(million)X 934(ones";)X 1187(A)X 1281(string)X 1513(is)X 1603(considered)X 2014(complex)X 2348(if)X 2432(it)X 2512(cannot)X 2777(be)X 2890(so)X 2998(easily)X 3234(described,)X 3623(like)X 3786(a)X 3855("ran-)X 576 2364(dom")N 801(string)X 1035(which)X 1283(does)X 1477(not)X 1623(follow)X 1886(any)X 2046(rule)X 2216(and)X 2376(hence)X 2614(we)X 2749(do)X 2870(not)X 3016(know)X 3244(how)X 3428(to)X 3530(describe)X 3856(apart)X 576 2524(from)N 771(giving)X 1021(it)X 1095(literally.)X 1417(A)X 1504(description)X 1920(of)X 2017(a)X 2080(string)X 2305(may)X 2481(depend)X 2759(on)X 2871(two)X 3027(things,)X 3289(the)X 3421(decoding)X 3767(method)X 576 2684(\(the)N 749(machine)X 1084(which)X 1335(interprets)X 1704(the)X 1848(description\))X 2305(and)X 2468(outside)X 2760(information)X 3214(available)X 3570(\(input)X 3818(to)X 3924(the)X 576 2844(machine\).)N 953(We)X 1102(are)X 1236(interested)X 1606(in)X 1702(descriptions)X 2155(which)X 2396(are)X 2529(effective,)X 2885(and)X 3038(restrict)X 3309(the)X 3443(decoders)X 3781(to)X 3876(Tur-)X 576 3004(ing)N 712(machines.)X 1112(Without)X 1425(loss)X 1585(of)X 1681(generality,)X 2079(our)X 2219(Turing)X 2482(machines)X 2838(use)X 2978(binary)X 3226(input)X 3432(strings)X 3691(which)X 3930(we)X 576 3164(call)N 2 f 743(programs)X 1 f 1084(.)X 1167(More)X 1396(formally,)X 1761(\256xing)X 2006(a)X 2084(Turing)X 2363(machine)X 2 f 2701(T)X 1 f 2750(,)X 2811(we)X 2952(would)X 3210(like)X 3381(to)X 3488(say)X 3643(that)X 2 f 3814(p)X 1 f 3896(is)X 3993(a)X 576 3324(description)N 998(of)X 2 f 1101(x)X 1 f 1170(if,)X 1277(on)X 1396(input)X 2 f 1609(p)X 1 f (,)S 2 f 1706(T)X 1 f 1786(outputs)X 2 f 2077(x)X 1 f 2116(.)X 2169(It)X 2254(is)X 2344(also)X 2517(convenient)X 2935(to)X 3035(allow)X 2 f 3262(T)X 1 f 3342(to)X 3442(have)X 3639(some)X 3856(extra)X 576 3484(information)N 2 f 1017(y)X 1 f 1080(to)X 1173(help)X 1349(to)X 1442(generate)X 2 f 1764(x)X 1 f 1803(.)X 1849(We)X 1995(write)X 2 f 2200(T)X 1 f 2263(\()X 2 f 2292(p)X 1 f (,)S 2 f 2365(y)X 1 f 2411(\))X 2 f 9 f 2440(=)X 2 f 2488(x)X 1 f 2551(to)X 2643(mean)X 2857(that)X 3013(Turing)X 3276(machine)X 2 f 3598(T)X 1 f 3670(with)X 3850(input)X 2 f 576 3644(p)N 1 f 642(and)X 2 f 791(y)X 1 f 852(terminates)X 1242(with)X 1421(output)X 2 f 1669(x)X 1 f 1708(.)X 3 f 776 3832(De\256nition)N 1170(1.)X 1 f 1258(The)X 1417(descriptional)X 1895(complexity)X 2 f 2315(C)X 8 s 2374 3849(T)N 1 f 11 s 2439 3832(of)N 2 f 2534(x)X 1 f 2573(,)X 2 f 2617(relative)X 1 f 2909(to)X 3000(Turing)X 3263(machine)X 2 f 3585(T)X 1 f 3657(and)X 3807(binary)X 576 3992(string)N 2 f 799(y)X 1 f 838(,)X 882(is)X 963(de\256ned)X 1244(by)X 2 f 1525 4232(C)N 8 s 1584 4249(T)N 1 f 11 s 1627 4232(\()N 2 f 1656(x)X 1 f 9 f 1716(|)X 2 f 1748(y)X 1 f 1794(\))X 2 f 9 f 1845(=)X 1 f 1915(min)X 2 f 2053({l)X 1 f 2127(\()X 2 f 2156(p)X 1 f 2207(\):)X 2 f 2275(p)X 9 f 2333(\316)X 2 f 2396({)X 1 f 2431(0,)X 2504(1)X 2 f (}*)S 1 f 2627(,)X 2 f 2685(T)X 1 f 2748(\()X 2 f 2777(p)X 1 f (,)S 2 f 2850(y)X 1 f 2896(\))X 2 f 9 f 2925(=)X 2 f 2973(x)X 3019(})X 1 f 3054(,)X 576 4472(or)N 9 f 671(\245)X 1 f 756(if)X 832(no)X 942(such)X 2 f 1125(p)X 1 f 1191(exists.)X 776 4660(The)N 938(complexity)X 1361(measure)X 1679(de\256ned)X 1963(above)X 2198(is)X 2282(useful)X 2522(and)X 2674(makes)X 2924(sense)X 3139(only)X 3321(if)X 3401(the)X 3535(complexity)X 3959(of)X 576 4820(a)N 638(string)X 862(does)X 1046(not)X 1182(depend)X 1458(on)X 1568(the)X 1698(choice)X 1950(of)X 2 f 2045(T)X 1 f 2094(.)X 2138(Therefore)X 2506(the)X 2636(following)X 3001(simple)X 3259(theorem)X 3570(is)X 3651(vital.)X 3875(This)X 576 4980(Invariance)N 982(Theorem)X 1335(is)X 1429(given)X 1660(by)X 1783(Solomonoff)X 2239([)X 2303(Solomonoff)X 2759(formal)X 3030(inductive)X 3395(inference)X 3758(Kolmo-)X 576 5140(gorov)N 803([)X 854(Kolmogorov)X 1326(Three)X 1553(approaches)X 1970(and)X 2119(Chaitin)X 2402([)X 2453(Chaitin)X 2736(Ch2)X 3 f 776 5328(Theorem)N 1150(1.)X 2 f 1270(There)X 1507(exists)X 1735(a)X 1811(universal)X 2171(Turing)X 2443(machine)X 2773(U,)X 2890(such)X 3084(that,)X 3277(for)X 3413(any)X 3573(other)X 3792(Turing)X 576 5488(machine)N 896(T,)X 989(there)X 1192(is)X 1273(a)X 1339(constant)X 1660(c)X 8 s 1699 5505(T)N 11 s 1764 5488(such)N 1947(that)X 2107(for)X 2232(all)X 2348(strings)X 2610(x)X 1 f 2649(,)X 2 f 2692(y,)X 2775(C)X 8 s 2834 5505(U)N 1 f 11 s 2887 5488(\()N 2 f 2916(x)X 1 f 9 f 2976(|)X 2 f 3008(y)X 1 f 3054(\))X 2 f 9 f 3105(\243)X 2 f 3175(C)X 8 s 3234 5505(T)N 1 f 11 s 3277 5488(\()N 2 f 3306(x)X 1 f 9 f 3366(|)X 2 f 3398(y)X 1 f 3444(\))X 2 f 9 f 3473(+)X 2 f 3521(c)X 8 s 3560 5505(T)N 11 s 3603 5488(.)N 776 5676(Proof.)N 1 f 1062(Fix)X 1221(some)X 1448(standard)X 1787(enumeration)X 2269(of)X 2383(Turing)X 2665(machines)X 2 f 3040(T)X 1 f 8 s 3099 5693(1)N 11 s 3138 5676(,)N 2 f 3167(T)X 1 f 8 s 3226 5693(2)N 11 s 3265 5676(,)N 3316 5650(.)N 3360(.)X 3404(.)X 3448 5676(.)N 3534(Let)X 2 f 3694(U)X 1 f 3799(be)X 3924(the)X 576 5836(Universal)N 945(Turing)X 1212(machine)X 1538(such)X 1726(that)X 1886(when)X 2103(starting)X 2395(on)X 2509(input)X 2717(0)X 2 f 8 s 5801(n)Y 1 f 11 s 2800 5836(1)N 2 f (p)S 1 f (,)S 2 f 2936(p)X 9 f 2994(\316)X 2 f 3057({)X 1 f 3092(0,)X 3165(1)X 2 f (}*)S 1 f 3288(,)X 2 f 3336(U)X 1 f 3425(simulates)X 3785(the)X 2 f 3919(n)X 1 f (th)S ep %%Page: 13 13 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2209 424(-)N 2260(13)X 2370(-)X 576 736(Turing)N 844(machine)X 2 f 1171(T)X 8 s 1220 753(n)N 1 f 11 s 1287 736(on)N 1403(input)X 2 f 1613(p)X 1 f (.)S 1707(For)X 1857(convenience)X 2325(in)X 2422(the)X 2558(proof,)X 2798(we)X 2928(choose)X 2 f 3200(U)X 1 f 3291(such)X 3481(that)X 3643(if)X 2 f 3726(T)X 8 s 3775 753(n)N 1 f 11 s 3843 736(halts,)N 576 896(then)N 2 f 751(U)X 1 f 837(\256rst)X 997(erases)X 1234(everything)X 1634(apart)X 1833(from)X 2027(the)X 2158(halting)X 2427(contents)X 2744(of)X 2 f 2840(T)X 8 s 2889 913(n)N 1 f 11 s 2928 896('s)N 3014(tape,)X 3205(and)X 3354(also)X 3518(halts.)X 3751(By)X 3876(con-)X 576 1056(struction,)N 939(for)X 1073(each)X 2 f 1266(p)X 9 f 1324(\316)X 2 f 1387({)X 1 f 1422(0,)X 1495(1)X 2 f (}*)S 1 f 1618(,)X 2 f 1672(T)X 8 s 1721 1073(n)N 1 f 11 s 1792 1056(started)N 2060(on)X 2 f 2181(p)X 1 f 2258(eventually)X 2659(halts)X 2859(iff)X 2 f 2975(U)X 1 f 3071(started)X 3339(on)X 3460(0)X 2 f 8 s 1021(n)Y 1 f 11 s 3543 1056(1)N 2 f (p)S 1 f 3664(eventually)X 576 1216(halts.)N 787(Choosing)X 2 f 1147(c)X 8 s 1186 1233(T)N 11 s 9 f 1229 1216(=)N 2 f 1277(n)X 9 f 1328(+)X 1 f 1376(1)X 1442(for)X 2 f 1566(T)X 8 s 1615 1233(n)N 1 f 11 s 1676 1216(\256nishes)N 1967(the)X 2097(proof.)X 10 f 2331(`)X 1 f 776 1404(Clearly,)N 1082(the)X 1214(Universal)X 1580(Turing)X 1844(machine)X 2 f 2167(U)X 1 f 2254(that)X 2411(satis\256es)X 2714(the)X 2846(Invariance)X 3241(Theorem)X 3584(is)X 2 f 3668(optimal)X 1 f 3963(in)X 576 1564(the)N 718(sense)X 942(that)X 2 f 1109(C)X 8 s 1168 1581(U)N 1 f 11 s 1255 1564(minorizes)N 1637(each)X 2 f 1832(C)X 8 s 1891 1581(T)N 1 f 11 s 1968 1564(up)N 2090(to)X 2193(a)X 2266(\256xed)X 2476(additive)X 2795(constant)X 3123(\(depending)X 3553(on)X 2 f 3675(U)X 1 f 3772(and)X 2 f 3932(T)X 1 f 3981(\).)X 576 1724(Moreover,)N 988(for)X 1134(each)X 1339(pair)X 1520(of)X 1637(Universal)X 2023(Turing)X 2307(machines)X 2 f 2684(U)X 1 f 2791(and)X 2 f 2962(U)X 1 f 9 f 3032(\242)X 1 f 3054(,)X 3120(satisfying)X 3508(the)X 3661(Invariance)X 576 1884(Theorem,)N 941(the)X 1074(complexities)X 1550(coincide)X 1873(up)X 1985(to)X 2078(an)X 2185(additive)X 2494(constant)X 2812(\(depending)X 3232(only)X 3413(on)X 2 f 3525(U)X 1 f 3612(and)X 2 f 3763(U)X 1 f 9 f 3833(\242)X 1 f 3855(\),)X 3930(for)X 576 2044(all)N 687(strings)X 2 f 944(x)X 1 f 983(,)X 2 f 1033(y)X 1 f 1072(:)X 9 f 1740 2284(|)N 2 f 1772(C)X 8 s 1831 2301(U)N 1 f 11 s 1884 2284(\()N 2 f 1913(x)X 1 f 9 f 1980(|)X 2 f 2026(y)X 1 f 2072(\))X 2 f 9 f 2115(-)X 2 f 2177(C)X 8 s 2236 2301(U)N 1 f 9 f 2287(\242)X 11 s 1 f 2310 2284(\()N 2 f 2339(x)X 1 f 9 f 2406(|)X 2 f 2452(y)X 1 f 2498(\))X 9 f 2541(|)X 2595(\243)X 2 f 2665(c)X 8 s 2704 2301(U)N 1 f 2750(,)X 2 f 2786(U)X 1 f 9 f 2837(\242)X 2 f 11 s 2860 2284(.)N 1 f 576 2524(Therefore,)N 966(we)X 1090(set)X 1210(the)X 1340(canonical)X 2 f 1701(conditional)X 2127(Kolmogorov)X 1 f 2590(complexity)X 2 f 3011(C)X 1 f 3084(\()X 2 f 3113(x)X 1 f 9 f 3173(|)X 2 f 3219(y)X 1 f 3265(\))X 3317(of)X 2 f 3413(x)X 1 f 3475(under)X 3698(condition)X 576 2684(of)N 2 f 671(y)X 1 f 732(equal)X 945(to)X 2 f 1036(C)X 8 s 1095 2701(U)N 1 f 11 s 1148 2684(\()N 2 f 1177(x)X 1 f 9 f 1237(|)X 2 f 1269(y)X 1 f 1315(\),)X 1388(for)X 1512(some)X 1720(\256xed)X 1918(optimal)X 2 f 2211(U)X 1 f 2274(.)X 2318(We)X 2462(call)X 2 f 2612(U)X 1 f 2697(the)X 2 f 2827(reference)X 1 f 3181(Turing)X 3443(machine.)X 3808(Hence)X 576 2844(the)N 708(Kolmogorov)X 1182(complexity)X 1604(of)X 1701(a)X 1764(string)X 1989(does)X 2174(not)X 2311(depend)X 2589(on)X 2701(the)X 2833(choice)X 3087(of)X 3184(encoding)X 3531(method)X 3821(and)X 3973(is)X 576 3004(well-de\256ned.)N 1090(De\256ne)X 1354(the)X 2 f 1492(unconditional)X 2013(Kolmogorov)X 1 f 2483(complexity)X 2911(of)X 2 f 3014(x)X 1 f 3082(as)X 2 f 3184(C)X 1 f 3257(\()X 2 f 3286(x)X 1 f 3332(\))X 2 f 9 f 3375(=)X 2 f 3437(C)X 1 f 3510(\()X 2 f 3539(x)X 1 f 9 f 3599(|)X 3631(e)X 1 f 3670(\),)X 3750(where)X 9 f 3993(e)X 1 f 576 3164(denotes)N 867(the)X 997(empty)X 1240(string)X 1463(\()X 2 f 1492(l)X 1 f 1531(\()X 9 f 1560(e)X 1 f 1599(\))X 2 f 9 f 1642(=)X 1 f 1704(0\).)X 3 f 776 3352(De\256nition)N 1175(2.)X 1 f 1269(In)X 1370(the)X 1506(sequel)X 1759(we)X 1889(need)X 2083(to)X 2180(use)X 2325(the)X 2 f 2461(pre\256x)X 1 f 2689(complexity)X 3115(variant,)X 3410(or)X 3511(self-delimiting)X 576 3512(complexity,)N 1020(rather)X 1249(than)X 2 f 1425(C)X 1 f 1498(\()X 2 f 1527(x)X 1 f 1573(\))X 1626(from)X 1821(De\256nition)X 2203(1.)X 2315(A)X 2 f 2402(pre\256x)X 2626(machine)X 1 f 2947(is)X 3029(a)X 3091(Turing)X 3354(machine)X 3676(with)X 3856(three)X 576 3672(tapes:)N 806(a)X 869(one-way)X 1195(input)X 1401(tape,)X 1594(a)X 1657(one-way)X 1983(output)X 2233(tape,)X 2426(and)X 2577(a)X 2640(two-way)X 2971(work)X 3175(tape.)X 3390(Initially,)X 3717(the)X 3850(input)X 576 3832(tape)N 751(contains)X 1073(an)X 1184(inde\256nitely)X 1615(long)X 1800(sequence)X 2150(of)X 2250(bits.)X 2427(If)X 2512(the)X 2647(machine)X 2973(halts,)X 3189(then)X 3368(the)X 3503(initial)X 3738(segment)X 576 3992(on)N 688(the)X 820(input)X 1027(tape)X 1199(it)X 1274(has)X 1416(read)X 1592(up)X 1705(till)X 1830(that)X 1988(time)X 2171(is)X 2255(considered)X 2661(the)X 2 f 2794(input)X 1 f 3001(or)X 2 f 3099(program)X 1 f 3406(,)X 3453(and)X 3605(the)X 3738(contents)X 576 4152(of)N 680(the)X 819(output)X 1076(tape)X 1254(is)X 1344(the)X 2 f 1483(output)X 1 f 1709(.)X 1783(Clearly,)X 2095(the)X 2233(set)X 2361(of)X 2464(programs)X 2826(of)X 2929(each)X 3120(such)X 3311(machine)X 3640(is)X 3729(a)X 3798(pre\256x-)X 576 4312(code.)N 811(\(Recall)X 1091(that)X 1249(if)X 2 f 1328(p)X 1 f 1397(and)X 2 f 1549(q)X 1 f 1618(are)X 1750(two)X 1907(code)X 2098(words)X 2337(of)X 2435(a)X 2 f 2499(pre\256x-code)X 1 f 2894(,)X 2941(then)X 2 f 3118(p)X 1 f 3187(is)X 3271(not)X 3409(a)X 3473(proper)X 3728(pre\256x)X 3959(of)X 2 f 576 4472(q)N 1 f (.\))S 722(We)X 873(can)X 1024(give)X 1205(an)X 1317(effective)X 1654(enumeration)X 2124(of)X 2226(all)X 2344(pre\256x)X 2577(machines)X 2938(in)X 3035(the)X 3171(standard)X 3497(way.)X 3715(Then)X 3924(the)X 2 f 576 4632(pre\256x)N 1 f 803(descriptional)X 1286(complexity)X 1711(of)X 2 f 1811(x)X 9 f 1864(\316)X 2 f 1927({)X 1 f 1962(0,)X 2035(1)X 2 f (}*)S 1 f 2158(,)X 2207(with)X 2391(respect)X 2667(to)X 2764(pre\256x)X 2997(machine)X 2 f 3324(T)X 1 f 3373(,)X 3423(and)X 3578(binary)X 3831(string)X 2 f 576 4792(y)N 1 f 615(,)X 659(is)X 740(de\256ned)X 1021(as)X 2 f 1514 5032(K)N 8 s 1573 5049(T)N 1 f 11 s 1616 5032(\()N 2 f 1645(x)X 1 f 9 f 1712(|)X 2 f 1758(y)X 1 f 1804(\))X 2 f 9 f 1855(=)X 1 f 1925(min)X 2 f 2063({l)X 1 f 2137(\()X 2 f 2166(p)X 1 f 2217(\):)X 2 f 2285(p)X 9 f 2343(\316)X 2 f 2406({)X 1 f 2441(0,)X 2514(1)X 2 f (}*)S 1 f 2637(,)X 2 f 2695(T)X 1 f 2758(\()X 2 f 2787(p)X 1 f (,)S 2 f 2860(y)X 1 f 2906(\))X 2 f 9 f 2935(=)X 2 f 2983(x)X 3029(})X 1 f 3064(,)X 576 5272(or)N 9 f 681(\245)X 1 f 776(if)X 862(such)X 2 f 1055(p)X 1 f 1131(do)X 1251(not)X 1397(exist.)X 1641(One)X 1820(can)X 1975(prove)X 2208(an)X 2324(Invariance)X 2728(Theorem)X 3079(for)X 3214(pre\256x)X 3452(complexity,)X 3905(and)X 576 5432(de\256ne)N 816(the)X 949(conditional)X 1372(and)X 1524(unconditional)X 2 f 2035(pre\256x)X 2260(Kolmogorov)X 2725(complexity)X 1 f 3107(,)X 3154(by)X 3266(\256xing)X 3496(some)X 3706(reference)X 576 5592(optimal)N 869(pre\256x)X 1096(machine,)X 1439(in)X 1530(exactly)X 1807(the)X 1937(same)X 2140(way)X 2308(as)X 2403(before,)X 2671(so)X 2771(we)X 2895(do)X 3005(not)X 3140(repeat)X 3377(the)X 3507(construction.)X 2 f 776 5780(Remark.)N 1 f 1098(The)X 1262(pre\256x)X 1494(Kolmogorov)X 1971(complexity)X 2397(of)X 2498(string)X 2 f 2727(x)X 1 f 2766(,)X 2816(is)X 2903(the)X 3039(length)X 3288(of)X 3389(the)X 3525(shortest)X 3827(pre\256x)X ep %%Page: 14 14 bp 11 s 0 xH 0 xS 1 f 10 s 0 32(--)N 4323(--)X 11 s 2209 424(-)N 2260(14)X 2370(-)X 576 736(program)N 907(that)X 1073(outputs)X 2 f 1366(x)X 1 f 1405(.)X 1482(In)X 1588(this)X 1749(exposition,)X 2172(we)X 2307(will)X 2478(use)X 2 f 2628(K)X 1 f 2701(\()X 2 f 2730(x)X 1 f 2776(\))X 2837(to)X 2938(denote)X 3205(the)X 3345(pre\256x)X 3582(Kolmogorov)X 576 896(complexity)N 1004(of)X 2 f 1107(x)X 1 f 1146(.)X 2 f 1220(C)X 1 f 1293(\()X 2 f 1322(x)X 1 f 1368(\))X 1427(and)X 2 f 1584(K)X 1 f 1657(\()X 2 f 1686(x)X 1 f 1732(\))X 1791(differ)X 2016(by)X 2135(at)X 2230(most)X 2433(a)X 2503(2)X 2561(log)X 2 f 2688(K)X 1 f 2761(\()X 2 f 2790(x)X 1 f 2836(\))X 2896(additive)X 3212(term.)X 3427(In)X 3531(some)X 3748(applica-)X 576 1056(tions)N 780(this)X 940(does)X 1133(not)X 1278(make)X 1501(any)X 1660(difference.)X 2070(But)X 2230(in)X 2331(some)X 2548(other)X 2760(applications,)X 3240(for)X 3373(example)X 3703(inductive)X 576 1216(inference,)N 950(this)X 1103(is)X 1187(vital.)X 1392(In)X 1490(particular,)X 1876(we)X 2004(need)X 2196(the)X 2330(property)X 2654(that)X 2813(the)X 2947(series)X 16 s 9 f 3173 1242(S)N 2 f 8 s 3249 1260(x)N 1 f 11 s 3298 1216(2)N 2 f 8 s 9 f 1181(-)Y 2 f 3377(K)X 1 f 3430(\()X 2 f 3451(x)X 1 f 3484(\))X 11 s 3538 1216(converges,)N 3942(cf.)X 576 1402(below.)N 3 f 776 1590(De\256nition)N 1170(3.)X 1 f 1258(A)X 1343(binary)X 1590(string)X 2 f 1813(x)X 1 f 1874(is)X 2 f 1955(incompressible)X 1 f 2510(if)X 2 f 2586(K)X 1 f 2659(\()X 2 f 2688(x)X 1 f 2734(\))X 2 f 9 f 2763(\263)X 2 f 2811(l)X 1 f 2850(\()X 2 f 2879(x)X 1 f 2925(\).)X 2 f 776 1778(Remark.)N 1 f 1096(Since)X 1318(Martin-L)X 1646 1726(.)N 1663(.)X 1641 1778(of)N 1740([)X 1795(Martin-L)X 2123 1726(.)N 2140(.)X 2118 1778(of)N 2217(de\256nition)X 2582(random)X 2877(1966)X 3079(has)X 3222(shown)X 3477(that)X 3636(incompres-)X 576 1938(sible)N 776(strings)X 1044(pass)X 1228(all)X 1350(effective)X 1691(statistical)X 2059(tests)X 2249(for)X 2384(randomness,)X 2859(we)X 2994(will)X 3164(also)X 3338(call)X 3498(incompressible)X 576 2098(strings)N 2 f 844(random)X 1 f 1150(strings.)X 1462(A)X 1558(simple)X 1827(counting)X 2169(argument)X 2535(shows)X 2787(that)X 2954(most)X 3160(strings)X 3429(are)X 3570(random.)X 3895(The)X 576 2258(theory)N 825(of)X 922(computabilit)X 1370(y)X 1438(shows)X 1681(that)X 1838(the)X 1970(function)X 2 f 2288(K)X 1 f 2361(\()X 2 f 2390(x)X 1 f 2436(\))X 2489(is)X 2571(noncomputable,)X 3160(but)X 3296(can)X 3441(be)X 3547(approximated)X 576 2418(to)N 671(any)X 824(degree)X 1084(of)X 1183(accuracy)X 1521(by)X 1635(a)X 1700(computable)X 2138(function.)X 2481(However,)X 2851(at)X 2942(no)X 3057(point)X 3266(in)X 3362(this)X 3517(approximation)X 576 2578(process)N 861(can)X 1005(we)X 1129(know)X 1346(the)X 1476(error.)X 1690(Cf.)X 1822(also)X 1986(the)X 2116(surveys)X 2406([)X 2457(Levin)X 2685(Zvonkin)X 3006(])X 3057([)X 3108(Li)X 3209(Two)X 3392(Decades)X 3 f 576 2898(2.3.)N 752(Semicomputable)X 1400(Functions)X 1794(and)X 1958(Measures)X 1 f 576 3086(We)N 724(consider)X 1048(recursive)X 1396(functions)X 1750(with)X 1933(values)X 2184(consisting)X 2569(of)X 2669(pairs)X 2867(of)X 2967(natural)X 3239(numbers.)X 3591(If)X 2 f 3676(
)X
1 f
3973(is)X
576 3246(such)N
768(a)X
838(value)X
1060(then)X
1243(we)X
1376(interpret)X
1706(this)X
1865(value)X
2086(as)X
2189(the)X
2327(rational)X
2627(number)X
2 f
2926(p)X
2977(/)X
3009(q)X
1 f
(,)S
3105(and)X
3262(say)X
3409(that)X
3572(the)X
3710(recursive)X
576 3406(function)N
892(is)X
2 f
973(rational)X
1280(valued.)X
3 f
776 3594(De\256nition.)N
1 f
1229(A)X
1329(real)X
1498(function)X
2 f
1836(f)X
1 f
1899(is)X
2 f
1996(semicomputable)X
2606(from)X
2810(below)X
1 f
3059(iff)X
3180(there)X
3394(exists)X
3633(a)X
3710(recursive)X
576 3754(function)N
2 f
898(g)X
1 f
956(\()X
2 f
985(x)X
1 f
1024(,)X
2 f
1074(k)X
1 f
1120(\))X
1177(with)X
1362(rational)X
1660(values)X
1913(\(or,)X
2065(equivalently,)X
2552(a)X
2619(computable)X
3059(real)X
3219(function)X
2 f
3541(g)X
1 f
3599(\()X
2 f
3628(x)X
1 f
3667(,)X
2 f
3717(k)X
1 f
3763(\)\),)X
3871(non-)X
576 3914(decreasing)N
976(in)X
2 f
1069(k)X
1 f
1108(,)X
1155(with)X
2 f
1344(f)X
1 f
1397(\()X
2 f
1426(x)X
1 f
1472(\))X
2 f
9 f
1515(=)X
1 f
1577(lim)X
2 f
8 s
1696 3931(k)N
9 f
1734 MX
(->)174 987 oc
1 f
9 f
1797(\245)X
2 f
11 s
1850 3914(g)N
1 f
1908(\()X
2 f
1937(x)X
1 f
1976(,)X
2 f
2026(k)X
1 f
2072(\).)X
2170(A)X
2258(function)X
2 f
2584(f)X
1 f
2634(is)X
2718(semicomputable)X
3322(from)X
3518(above,)X
3775(if)X
2 f
9 f
3854(-)X
2 f
3923(f)X
1 f
3973(is)X
576 4074(semicomputable)N
1177(from)X
1370(below.)X
776 4262(\(An)N
951(equivalent)X
1358(de\256nition:)X
2 f
1768(f)X
1 f
1832(is)X
1930(a)X
2008(function)X
2341(that)X
2513(is)X
2611(semicomputable)X
3229(from)X
3439(below)X
3693(if)X
3786(the)X
3934(set)X
2 f
576 4422({)N
1 f
611(\()X
2 f
640(x)X
1 f
679(,)X
2 f
729(r)X
1 f
770(\):)X
2 f
838(r)X
1 f
9 f
893(\243)X
2 f
962(f)X
1 f
1015(\()X
2 f
1044(x)X
1 f
1090(\),)X
2 f
1162(r)X
1 f
1218(is)X
1299(rational)X
2 f
1569(})X
1 f
1626(is)X
1707(recursively)X
2120(enumerable.\))X
776 4610(A)N
863(real)X
1019(function)X
2 f
1344(f)X
1 f
1393(is)X
1476(computable)X
1912(iff)X
2019(there)X
2219(is)X
2302(a)X
2366(recursive)X
2713(function)X
2 f
3032(g)X
1 f
3090(\()X
2 f
3119(x)X
1 f
3158(,)X
2 f
3208(k)X
1 f
3254(\))X
3308(with)X
3490(rational)X
3785(values,)X
576 4770(and)N
9 f
739(|)X
2 f
778(f)X
1 f
831(\()X
2 f
860(x)X
1 f
906(\))X
2 f
9 f
949(-)X
2 f
1011(g)X
1 f
1069(\()X
2 f
1098(x)X
1 f
1137(,)X
2 f
1187(k)X
1 f
1233(\))X
9 f
1276(|)X
2 f
1322(<)X
1 f
1395(1)X
2 f
1446(/)X
1478(k)X
1 f
1517(.)X
776 4958(Obviously,)N
1197(all)X
1318(recursive)X
1672(functions)X
2032(are)X
2171(computable,)X
2637(and)X
2796(all)X
2917(computable)X
3361(functions)X
3721(are)X
3861(sem-)X
576 5118(icomputable.)N
1090(However,)X
1466(not)X
1612(all)X
1734(semicomputable)X
2346(functions)X
2707(are)X
2847(computable,)X
3313(and)X
3472(not)X
3617(all)X
3738(comput-)X
576 5278(able)N
761(functions)X
1127(are)X
1272(recursive.)X
1676(Nontrivial)X
2077(examples)X
2448(of)X
2559(functions)X
2925(that)X
3097(are)X
3243(semicomputable)X
3861(from)X
576 5438(above)N
808(but)X
943(not)X
1078(computable)X
1512(are)X
2 f
1641(C)X
1 f
1714(\()X
2 f
1743(x)X
1 f
1789(\),)X
2 f
1862(C)X
1 f
1935(\()X
2 f
1964(x)X
1 f
9 f
2031(|)X
2 f
2077(y)X
1 f
2123(\),)X
2 f
2196(K)X
1 f
2269(\()X
2 f
2298(x)X
1 f
2344(\),)X
2417(and)X
2 f
2566(K)X
1 f
2639(\()X
2 f
2668(x)X
1 f
9 f
2735(|)X
2 f
2781(y)X
1 f
2827(\).)X
776 5626(The)N
939(following)X
1308(analysis)X
1618(is)X
1703(a)X
1768(simpli\256ed)X
2148(version)X
2434(over)X
2617(the)X
2752(discrete)X
3053(space)X
2 f
3275(N)X
1 f
3361(\(or)X
3490(the)X
3625(set)X
3750(of)X
3850(\256nite)X
576 5786(binary)N
827(strings\),)X
1139(of)X
1238(Zvonkin)X
1563(and)X
1716(Levin)X
1948([)X
2003(Levin)X
2235(Zvonkin)X
2560(We)X
2708(follow)X
2963(to)X
3057(some)X
3268(extent)X
3509([)X
3563(G)X
3641(\302)X
3626(acs)X
3763(Lecture)X
ep
%%Page: 15 15
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(15)X
2370(-)X
576 736(Notes)N
808(Functions)X
9 f
1183(m)X
1 f
1234(:)X
2 f
1273(N)X
1 f
9 f
1353 MX
(->)174 987 oc
1 f
1454([0,)X
1570(1])X
1670(that)X
1830(satisfy)X
2087(the)X
2222(usual)X
2435(properties)X
2815(of)X
2916(probability)X
3332(distributions)X
3802(except)X
576 896(that)N
16 s
9 f
2092 1162(S)N
2 f
8 s
2168 1180(x)N
1 f
11 s
9 f
2203 1136(m)N
1 f
2254(\()X
2 f
2283(x)X
1 f
2329(\))X
9 f
2380(\243)X
1 f
2450(1)X
2 f
(.)S
1 f
576 1402(we)N
710(shall)X
909(call)X
2 f
1069(measures)X
1 f
1400(.)X
1476(We)X
1630(say)X
1779(that)X
1944(a)X
2016(measure)X
9 f
2342(m)X
1 f
2426(\(multipli)X
2737(catively\))X
2 f
3079(dominates)X
1 f
3474(a)X
3546(measure)X
9 f
3872(m\242)X
1 f
3978(if)X
576 1562(there)N
776(exists)X
1001(a)X
1064(constant)X
2 f
1382(c)X
1 f
1445(such)X
1630(that,)X
1809(for)X
1935(all)X
2 f
2047(x)X
1 f
2109(in)X
2 f
2201(N)X
1 f
2260(,)X
2305(we)X
2430(have)X
9 f
2619(m\242)X
1 f
2692(\()X
2 f
2721(x)X
1 f
2767(\))X
9 f
2810(\243)X
2 f
2872(c)X
1 f
9 f
2932(m)X
1 f
2983(\()X
2 f
3012(x)X
1 f
3058(\).)X
3154(It)X
3231(is)X
3313(known)X
3575(from)X
3769(the)X
3900(cal-)X
576 1722(culus)N
791(that)X
2 f
953(no)X
1 f
1070(measure)X
9 f
1392(m)X
1 f
1472(dominates)X
1864(all)X
1982(measures:)X
2363(for)X
2494(each)X
2684(measure)X
9 f
3006(m)X
1 f
3086(there)X
3291(is)X
3379(a)X
3447(measure)X
9 f
3769(m\242)X
1 f
3871(such)X
576 1882(that)N
739(lim)X
9 f
872(m\242)X
1 f
945(\()X
2 f
974(x)X
1 f
1020(\))X
2 f
1056(/)X
1 f
9 f
1088(m)X
1 f
1139(\()X
2 f
1168(x)X
1 f
1214(\))X
2 f
9 f
1257(=)X
1 f
9 f
1319(\245)X
1 f
1382(.)X
1456(However,)X
1828(if)X
1911(we)X
2042(restrict)X
2316(ourselves)X
2677(to)X
2775(the)X
2912(class)X
3112(of)X
3214(semicomputable)X
3822(meas-)X
576 2042(ures,)N
766(then)X
940(it)X
1012(turns)X
1210(out)X
1346(that)X
1502(this)X
1653(class)X
1847(contains)X
2164(an)X
2270(`absorbing')X
2698(element,)X
3023(a)X
3085(measure)X
3401(that)X
3557(dominates)X
3943(all)X
576 2202(measures)N
932(in)X
1030(the)X
1167(class.)X
1410(We)X
1560(call)X
1716(the)X
1852(measure)X
2173(that)X
2334(dominates)X
2725(all)X
2842(other)X
3051(measures)X
3406(in)X
3503(a)X
3570(given)X
3794(class)X
3993(a)X
2 f
576 2362(universal)N
930(measure)X
1 f
1253(for)X
1381(that)X
1540(class.)X
1781(This)X
1964(important)X
2334(observation)X
2771(that)X
2930(such)X
3118(a)X
3184(measure)X
3504(exists)X
3732(was)X
3895(\256rst)X
576 2522(made)N
789(by)X
899(Levin)X
1127([)X
1178(Levin)X
1406(Zvonkin)X
3 f
776 2710(Theorem)N
1142(2.)X
2 f
1254(The)X
1410(class)X
1610(of)X
1703(measures)X
2058(that)X
2220(are)X
2361(semicomputable)X
2958(from)X
3149(below)X
3385(contains)X
3709(a)X
3778(univer-)X
576 2870(sal)N
701(measure.)X
776 3058(Proof.)N
1 f
1072(First)X
1285(we)X
1438(consider)X
1787(the)X
1946(standard)X
2295(enumeration)X
2787(of)X
2911(all)X
3052(partial)X
3330(recursive)X
3704(functions)X
9 f
576 3218(f)N
8 s
1 f
622 3235(1)N
11 s
661 3218(,)N
9 f
704(f)X
8 s
1 f
750 3235(2)N
11 s
789 3218(,)N
2 f
818(...)X
1 f
(.)S
952(Each)X
9 f
1152(f)X
2 f
9 f
1212(=)X
1 f
9 f
1274(f)X
2 f
8 s
1320 3235(i)N
1 f
11 s
1369 3218(in)N
1462(this)X
1614(list)X
1747(is)X
1830(a)X
1893(function)X
2211(on)X
2323(the)X
2455(positive)X
2759(integers.)X
3106(Let)X
2 f
3248(<)X
1 f
3307 3192(.)N
2 f
3218(>)Y
1 f
3412(denote)X
3671(a)X
3734(standard)X
576 3378(effective)N
918(invertible)X
1291(pairing)X
1576(function)X
1905(over)X
2 f
2096(N)X
1 f
2190(to)X
2294(associate)X
2647(a)X
2721(unique)X
2996(natural)X
3276(number)X
2 f
3580( )X
1 f
1740(,)X
2 f
1783(k)X
9 f
1843(=)X
1 f
1905(1,)X
1978(2,)X
2051(...)X
2 f
(}.)S
1 f
576 4898(The)N
737(resulting)X
2 f
1070(s)X
1 f
1104(-enumeration)X
1599(contains)X
1918(all)X
2032(semicomputable)X
2636(functions.)X
3033(Next)X
3229(we)X
3356(use)X
3498(each)X
3684(semicom-)X
576 5058(putable)N
865(function)X
2 f
1188(s)X
1 f
1251(to)X
1349(compute)X
1682(a)X
1749(measure)X
9 f
2070(m)X
1 f
2149(from)X
2348(below.)X
2613(Initially,)X
2944(set)X
9 f
3070(m)X
1 f
3121(\()X
2 f
3150(x)X
1 f
3196(\))X
2 f
9 f
3239(=)X
1 f
3301(0)X
3373(for)X
3503(all)X
2 f
3620(x)X
1 f
3659(.)X
3709(If)X
2 f
3795(s)X
1 f
3843(\(1\))X
3973(is)X
576 5218(unde\256ned)N
946(then)X
9 f
1121(m)X
1 f
1195(will)X
1356(not)X
1492(change)X
1764(any)X
1914(more)X
2118(and)X
2268(it)X
2341(is)X
2423(trivially)X
2727(a)X
2790(measure.)X
3129(Otherwise,)X
3536(for)X
2 f
3662(k)X
9 f
3722(=)X
1 f
3784(1,)X
3871(2,)X
3944(...,)X
576 5378(if)N
2 f
664(s)X
8 s
708 5343(k)N
1 f
11 s
743 5378(\(1\))N
2 f
9 f
859(+)X
2 f
921(s)X
8 s
965 5343(k)N
1 f
11 s
1000 5378(\(2\))N
2 f
9 f
1116(+)X
1 f
1164(...)X
2 f
9 f
(+)S
2 f
1292(s)X
8 s
1336 5343(k)N
1 f
11 s
1371 5378(\()N
2 f
1400(k)X
1 f
1446(\))X
9 f
1489(\243)X
1 f
1551(1)X
1629(then)X
1815(set)X
9 f
1947(m)X
1 f
1998(\()X
2 f
2027(i)X
1 f
2059(\))X
2102(:)X
2 f
9 f
2127(=)X
2 f
2189(s)X
8 s
2233 5343(k)N
1 f
11 s
2268 5378(\()N
2 f
2297(i)X
1 f
2329(\))X
2391(for)X
2 f
2526(i)X
9 f
2572(=)X
1 f
2634(1,)X
2721(2,)X
2794(...,)X
2 f
2903(k)X
1 f
2942(,)X
2997(else)X
3167(the)X
3308(computation)X
3783(of)X
9 f
3889(m)X
1 f
3973(is)X
576 5538(\256nished.)N
776 5726(There)N
1006(are)X
1138(three)X
1339(mutually)X
1679(exclusive)X
2037(ways)X
2242(the)X
2375(computation)X
2842(of)X
9 f
2940(m)X
1 f
3016(can)X
3163(go,)X
3299(exhausting)X
3707(all)X
3822(possi-)X
576 5886(bilities.)N
886(Firstly,)X
2 f
1163(s)X
1 f
1220(is)X
1302(already)X
1584(a)X
1646(measure)X
1962(and)X
9 f
2112(m)X
1 f
2177(:)X
2 f
9 f
2202(=)X
2 f
2264(s)X
1 f
2298(.)X
2365(Secondly,)X
2738(for)X
2863(some)X
2 f
3072(x)X
1 f
3134(and)X
2 f
3284(k)X
1 f
3346(with)X
2 f
3526(x)X
1 f
9 f
3586(\243)X
2 f
3648(k)X
1 f
3710(the)X
3841(value)X
ep
%%Page: 16 16
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(16)X
2370(-)X
2 f
576 736(s)N
8 s
620 701(k)N
1 f
11 s
655 736(\()N
2 f
684(x)X
1 f
730(\))X
805(is)X
910(unde\256ned.)X
1325(Then)X
1552(the)X
1706(values)X
1977(of)X
9 f
2096(m)X
1 f
2194(do)X
2329(not)X
2489(change)X
2785(any)X
2959(more)X
3187(from)X
9 f
3405(m)X
1 f
3456(\()X
2 f
3485(i)X
1 f
3517(\))X
2 f
9 f
3560(=)X
2 f
3622(s)X
8 s
3666 701(k)N
9 f
3709(-)X
1 f
3754(1)X
11 s
3793 736(\()N
2 f
3822(i)X
1 f
3854(\))X
3930(for)X
2 f
576 896(i)N
9 f
622(=)X
1 f
684(1,)X
771(2,)X
844(...,)X
2 f
953(k)X
9 f
1006(-)X
1 f
1054(1,)X
1146(and)X
9 f
1299(m)X
1 f
1350(\()X
2 f
1379(i)X
1 f
1411(\))X
2 f
9 f
1454(=)X
1 f
1516(0)X
1586(for)X
2 f
1714(i)X
1 f
9 f
1760(\263)X
2 f
1822(k)X
1 f
1861(,)X
1909(even)X
2101(though)X
2372(the)X
2505(computation)X
2972(of)X
9 f
3070(m)X
1 f
3146(goes)X
3332(on)X
3445(forever.)X
3745(Thirdly,)X
576 1056(there)N
776(is)X
859(a)X
922(\256rst)X
2 f
1083(k)X
1 f
1146(such)X
1331(that)X
2 f
1488(s)X
8 s
1532 1021(k)N
1 f
11 s
1567 1056(\(1\))N
2 f
9 f
1683(+)X
2 f
1745(s)X
8 s
1789 1021(k)N
1 f
11 s
1824 1056(\(2\))N
2 f
9 f
1940(+)X
1 f
1988(...)X
2 f
9 f
(+)S
2 f
2116(s)X
8 s
2160 1021(k)N
1 f
11 s
2195 1056(\()N
2 f
2224(k)X
1 f
2270(\))X
2 f
2313(>)X
1 f
2386(1,)X
2476(that)X
2633(is,)X
2739(the)X
2872(new)X
3043(approximation)X
3583(of)X
9 f
3681(m)X
1 f
3757(violates)X
576 1216(the)N
714(condition)X
1078(of)X
1181(measure.)X
1525(Then)X
1735(the)X
1872(approximation)X
2416(of)X
9 f
2518(m)X
1 f
2598(is)X
2686(\256nished)X
2994(as)X
3096(in)X
3194(the)X
3331(second)X
3604(case.)X
3806(But)X
3963(in)X
576 1376(this)N
726(case)X
899(the)X
1029(algorithm)X
1395(terminates,)X
1807(and)X
9 f
1956(m)X
1 f
2029(is)X
2110(even)X
2298(computable.)X
776 1564(Thus,)N
1000(the)X
1134(above)X
1370(procedure)X
1747(yields)X
1984(an)X
2093(effective)X
2427(enumeration)X
9 f
2894(m)X
8 s
1 f
2945 1581(1)N
11 s
2984 1564(,)N
9 f
3027(m)X
8 s
1 f
3078 1581(2)N
11 s
3117 1564(,)N
2 f
3146(...)X
1 f
3238(of)X
3337(all)X
3453(semicomputable)X
576 1724(measures.)N
969(De\256ne)X
1225(the)X
1355(function)X
9 f
1671(m)X
8 s
1 f
1722 1741(0)N
11 s
1783 1724(as:)N
9 f
1922 1964(m)N
8 s
1 f
1973 1981(0)N
11 s
2012 1964(\()N
2 f
2041(x)X
1 f
2087(\))X
2 f
9 f
2138(=)X
1 f
16 s
9 f
2208 1990(S)N
2 f
8 s
2284 2008(n)N
1 f
11 s
2337 1964(2)N
2 f
8 s
9 f
1929(-)Y
2 f
2416(n)X
1 f
11 s
9 f
2469 1964(m)N
2 f
8 s
2520 1981(n)N
1 f
11 s
2559 1964(\()N
2 f
2588(x)X
1 f
2634(\).)X
576 2230(It)N
652(follows)X
938(that)X
9 f
1093(m)X
8 s
1 f
1144 2247(0)N
11 s
1205 2230(is)N
1286(a)X
1347(measure)X
1662(since)X
16 s
9 f
1567 2496(S)N
2 f
8 s
1643 2514(x)N
1 f
11 s
9 f
1678 2470(m)N
8 s
1 f
1729 2487(0)N
11 s
1768 2470(\()N
2 f
1797(x)X
1 f
1843(\))X
2 f
9 f
1894(=)X
1 f
16 s
9 f
1964 2496(S)N
2 f
8 s
2040 2514(n)N
1 f
11 s
2093 2470(2)N
2 f
8 s
9 f
2435(-)Y
2 f
2172(n)X
1 f
16 s
9 f
2225 2496(S)N
2 f
8 s
2301 2514(x)N
1 f
11 s
9 f
2350 2470(m)N
2 f
8 s
2401 2487(n)N
1 f
11 s
2440 2470(\()N
2 f
2469(x)X
1 f
2515(\))X
9 f
2566(\243)X
16 s
2636 2496(S)N
2 f
8 s
2712 2514(n)N
1 f
11 s
2765 2470(2)N
2 f
8 s
9 f
2435(-)Y
2 f
2844(n)X
1 f
11 s
2 f
9 f
2905 2470(=)N
1 f
2975(1.)X
576 2736(The)N
739(function)X
9 f
1059(m)X
8 s
1 f
1110 2753(0)N
11 s
1175 2736(is)N
1261(also)X
1430(semicomputable)X
2036(from)X
2234(below,)X
2498(since)X
9 f
2706(m)X
2 f
8 s
2757 2753(n)N
1 f
11 s
2796 2736(\()N
2 f
2825(x)X
1 f
2871(\))X
2927(is)X
3013(semicomputable)X
3619(from)X
3817(below)X
576 2896(in)N
2 f
672(n)X
1 f
743(and)X
2 f
897(x)X
1 f
936(.)X
985(\(Use)X
1177(the)X
1312(universal)X
1662(partial)X
1915(recursive)X
2264(function)X
9 f
2585(f)X
8 s
1 f
2631 2913(0)N
11 s
2697 2896(and)N
2851(the)X
2986(construction)X
3449(above.\))X
3759(Finally,)X
9 f
576 3056(m)N
8 s
1 f
627 3073(0)N
11 s
688 3056(dominates)N
1073(each)X
9 f
1257(m)X
2 f
8 s
1308 3073(n)N
1 f
11 s
1370 3056(since)N
9 f
1574(m)X
8 s
1 f
1625 3073(0)N
11 s
1664 3056(\()N
2 f
1693(x)X
1 f
1739(\))X
2 f
1782(>)X
1 f
1855(2)X
2 f
8 s
9 f
3021(-)Y
2 f
1934(n)X
1 f
11 s
9 f
1987 3056(m)N
2 f
8 s
2038 3073(n)N
1 f
11 s
2077 3056(\()N
2 f
2106(x)X
1 f
2152(\).)X
2226(Therefore,)X
9 f
2617(m)X
8 s
1 f
2668 3073(0)N
11 s
2730 3056(is)N
2812(a)X
2874(universal)X
3220(semicomputable)X
3822(meas-)X
576 3216(ure.)N
10 f
732(`)X
1 f
776 3404(Obviously,)N
1194(there)X
1400(are)X
1537(countably)X
1915(in\256nite)X
2196(universal)X
2549(semicomputable)X
3158(measures.)X
3537(We)X
3689(now)X
3870(\256x)X
3993(a)X
2 f
576 3564(reference)N
1 f
938(universal)X
1291(semicomputable)X
1900(measure)X
9 f
2223(m)X
8 s
1 f
2274 3581(0)N
11 s
2313 3564(\()N
2 f
2342(x)X
1 f
2388(\),)X
2469(and)X
2626(denote)X
2890(it)X
2969(by)X
3 f
3086(m)X
1 f
3159(\()X
2 f
3188(x)X
1 f
3234(\).)X
3336(It)X
3419(will)X
3586(turn)X
3757(out)X
3899(that)X
576 3724(function)N
3 f
892(m)X
1 f
965(\()X
2 f
994(x)X
1 f
1040(\))X
1091(adequately)X
1495(captures)X
1810(Solomonoff's)X
2316(envisioned)X
2720(universal)X
2 f
3065(a)X
3131(priori)X
1 f
3359(probability.)X
776 3912(If)N
880(a)X
965(semicomputable)X
1590(measure)X
1929(is)X
2034(also)X
2222(a)X
2307(probability)X
2741(distribution)X
3196(then)X
3395(it)X
3492(is)X
3598(computable.)X
576 4072(Namely,)N
933(if)X
1043(we)X
1201(compute)X
1561(an)X
1700(approximation)X
9 f
2271(m)X
2 f
8 s
2322 4037(k)N
1 f
11 s
2413 4072(of)N
2542(the)X
2706(function)X
9 f
3056(m)X
1 f
3163(from)X
3390(below)X
3660(for)X
3817(which)X
16 s
9 f
576 4258(S)N
2 f
8 s
652 4276(x)N
1 f
11 s
9 f
701 4232(m)N
2 f
8 s
752 4197(k)N
1 f
11 s
787 4232(\()N
2 f
816(x)X
1 f
862(\))X
2 f
913(>)X
1 f
994(1)X
2 f
9 f
1052(-)X
1 f
9 f
1114(e)X
1 f
1153(,)X
1197(then)X
1371(we)X
1495(have)X
9 f
1697(|)X
1729(m)X
1 f
1780(\()X
2 f
1809(x)X
1 f
1855(\))X
2 f
9 f
1898(-)X
1 f
9 f
1960(m)X
2 f
8 s
2011 4197(k)N
1 f
11 s
2046 4232(\()N
2 f
2075(x)X
1 f
2121(\))X
9 f
2164(|)X
2 f
2210(<)X
1 f
9 f
2283(e)X
1 f
2344(for)X
2468(all)X
2 f
2579(x)X
1 f
2618(.)X
776 4446(Any)N
954(positive)X
1261(function)X
2 f
1582(w)X
1 f
1655(\()X
2 f
1684(x)X
1 f
1730(\))X
1786(such)X
1974(that)X
16 s
9 f
2135 4472(S)N
2 f
8 s
2211 4490(x)N
11 s
2246 4446(w)N
1 f
2319(\()X
2 f
2348(x)X
1 f
2394(\))X
9 f
2437(\243)X
1 f
2499(1)X
2571(must)X
2771(converge)X
3121(to)X
3218(zero.)X
3419(Hence)X
3 f
3671(m)X
1 f
3744(\()X
2 f
3773(x)X
1 f
3819(\))X
3876(con-)X
576 4632(verges)N
832(to)X
928(zero)X
1106(as)X
1206(well.)X
1407(However,)X
1777(it)X
1854(converges)X
2237(to)X
2333(zero)X
2511(slower)X
2772(than)X
2950(any)X
3103(positive)X
3409(computable)X
3847(func-)X
576 4792(tion)N
739(that)X
897(converges)X
1278(to)X
1372(zero.)X
1592(That)X
1779(is,)X
3 f
1885(m)X
1 f
1958(\()X
2 f
1987(x)X
1 f
2033(\))X
2087(is)X
2171(not)X
2310(computable,)X
2770(and)X
2923(therefore)X
3266(it)X
3342(is)X
3427(not)X
3566(a)X
3631(proper)X
3886(pro-)X
576 4952(bability)N
886(distribution:)X
16 s
9 f
1358 4978(S)N
2 f
8 s
1434 4996(x)N
3 f
11 s
1469 4952(m)N
1 f
1542(\()X
2 f
1571(x)X
1 f
1617(\))X
2 f
1660(<)X
1 f
1733(1.)X
1838(There)X
2082(is)X
2180(no)X
2306(analogous)X
2701(result)X
2935(to)X
3042(Theorem)X
3398(2)X
3480(for)X
3620(computable)X
576 5138(measures:)N
951(amongst)X
1273(all)X
1385(computable)X
1820(measures)X
2170(there)X
2369(is)X
2451(no)X
2562(universal)X
2908(one.)X
3102(This)X
3282(fact)X
3437(is)X
3519(one)X
3669(of)X
3765(the)X
3896(rea-)X
576 5298(sons)N
754(for)X
878(introducing)X
1307(the)X
1437(notion)X
1685(of)X
1780(semicomputable)X
2381(measures.)X
ep
%%Page: 17 17
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(17)X
2370(-)X
3 f
576 736(2.4.)N
752(The)X
921(Solomonoff-Levin)X
1618(Distribution)X
1 f
576 924(The)N
756(original)X
1074(incentive)X
1441(to)X
1553(develop)X
1875(a)X
1957(theory)X
2225(of)X
2341(algorithmic)X
2792(information)X
3252(content)X
3556(of)X
3673(individual)X
576 1084(objects)N
865(was)X
1039(Ray)X
1219(Solomonoff's)X
1741(invention)X
2113(of)X
2224(a)X
2 f
2301(universal)X
2667(a)X
2749(priori)X
1 f
2993(probability)X
3419(that)X
3590(can)X
3750(be)X
3871(used)X
576 1244(instead)N
858(of)X
963(the)X
1103(actual)X
1346(\(but)X
1520(unknown\))X
2 f
1908(a)X
1984(priori)X
1 f
2222(probability)X
2642(in)X
2743(applying)X
3084(Bayes')X
3360(Rule.)X
3603(His)X
3757(original)X
576 1404(suggestion)N
980(was)X
1143(to)X
1239(set)X
1364(the)X
2 f
1499(a)X
1570(priori)X
1 f
1803(probability)X
2 f
2218(P)X
1 f
2286(\()X
2 f
2315(x)X
1 f
2361(\))X
2417(of)X
2517(a)X
2583(\256nite)X
2792(binary)X
3044(string)X
2 f
3272(x)X
1 f
3338(to)X
16 s
9 f
3434 1430(S)N
11 s
1 f
3510 1404(2)N
2 f
8 s
9 f
1369(-)Y
2 f
3589(l)X
1 f
3617(\()X
2 f
3638(p)X
1 f
3675(\))X
11 s
3703 1404(,)N
3751(the)X
3885(sum)X
576 1572(taken)N
794(over)X
977(all)X
1093(programs)X
2 f
1452(p)X
1 f
1523(with)X
2 f
1707(U)X
1 f
1784(\()X
2 f
1813(p)X
1 f
1864(\))X
2 f
9 f
1907(=)X
2 f
1969(x)X
1 f
2036(where)X
2 f
2278(U)X
1 f
2369(is)X
2456(the)X
2592(reference)X
2946(Turing)X
3214(machine)X
3541(of)X
3642(Theorem)X
3988(1)X
576 1732(for)N
713(the)X
2 f
856(C)X
1 f
915(-complexity.)X
1421(However,)X
1799(using)X
2025(plain)X
2237(Turing)X
2512(machines)X
2879(this)X
3041(is)X
3134(improper,)X
3513(since)X
3728(not)X
3875(only)X
576 1892(does)N
16 s
9 f
764 1918(S)N
2 f
8 s
840 1936(x)N
11 s
875 1892(P)N
1 f
943(\()X
2 f
972(x)X
1 f
1018(\))X
1074(diverge,)X
1387(but)X
1527(for)X
1656(some)X
2 f
1869(x)X
1 f
1935(even)X
2 f
2128(P)X
1 f
2196(\()X
2 f
2225(x)X
1 f
2271(\))X
2327(itself)X
2531(diverges.)X
2900(To)X
3025(counteract)X
3420(this)X
3576(defect,)X
3841(Solo-)X
576 2078(monoff)N
858(in)X
950(1960)X
1149(and)X
1299(1964)X
1498(used)X
1682(normalizing)X
2132(terms,)X
2372(but)X
2507(the)X
2637(overall)X
2904(result)X
3122(was)X
3280(unconvincing.)X
3826(Levin)X
576 2238([)N
630(Levin)X
861(Zvonkin)X
1185(succeeded)X
1571(in)X
1666(1970)X
1868(to)X
1963(\256nd)X
2126(a)X
2191(proper)X
2446(mathematical)X
2949(expression)X
3351(of)X
3450(the)X
2 f
3584(a)X
3654(priori)X
1 f
3886(pro-)X
576 2398(bability,)N
892(of)X
988(which)X
1226(we)X
1351(present)X
1628(the)X
1759(simpler)X
2047(version)X
2329(over)X
2508(the)X
2639(discrete)X
2936(domain)X
2 f
3224(N)X
1 f
3283(.)X
3328(This)X
3507(was)X
3665(elaborated)X
576 2558(by)N
691(Levin)X
924(in)X
1020(1973)X
1223(and)X
1377(1974)X
1580([)X
1636(Levin)X
1869(Notion)X
2141(random)X
2437(])X
2493([)X
2549(Levin)X
2782(non-growth)X
3219(and)X
3373(Levin)X
3606(and)X
3760(G)X
3838(\302)X
3823(acs)X
3963(in)X
576 2718(1974)N
789([)X
855(G)X
933(\302)X
918(acs)X
1067(symmetry)X
1456(and)X
1619(independently)X
2155(by)X
2279(Chaitin)X
2576(in)X
2681(1975)X
2893([)X
2958(Chaitin)X
3255(theory)X
3516(1975)X
3728(formally)X
576 2878(identical)N
3 f
776 3066(De\256nition.)N
1 f
1215(The)X
2 f
1375(Solomonoff-Levin)X
2026(distribution)X
1 f
2463(\(actually)X
2796(a)X
2859(measure\))X
3205(on)X
3317(the)X
3449(positive)X
3753(integers)X
576 3226(is)N
659(de\256ned)X
942(by)X
2 f
1054(P)X
8 s
1108 3243(U)N
1 f
11 s
1161 3226(\()N
2 f
1190(x)X
1 f
1236(\))X
2 f
9 f
1279(=)X
1 f
16 s
9 f
1341 3252(S)N
11 s
1 f
1417 3226(2)N
2 f
8 s
9 f
3191(-)Y
2 f
1496(l)X
1 f
1524(\()X
2 f
1545(p)X
1 f
1582(\))X
11 s
1610 3226(,)N
1656(where)X
1894(the)X
2026(sum)X
2197(is)X
2280(taken)X
2495(over)X
2675(all)X
2788(programs)X
2 f
3144(p)X
1 f
3212(for)X
3337(which)X
3575(the)X
3706(reference)X
576 3394(pre\256x-machine)N
2 f
1131(U)X
1 f
1216(of)X
1311(Theorem)X
1651(1)X
1717(outputs)X
2 f
1999(x)X
1 f
2038(.)X
2104(This)X
2283(is)X
2364(a)X
2425(measure)X
2740(because)X
3040(of)X
3135(the)X
3265(following.)X
3 f
776 3588(Kraft's)N
1075(Inequality.)X
1 f
1533(If)X
2 f
1618(l)X
1 f
8 s
1653 3605(1)N
11 s
1692 3588(,)N
2 f
1721(l)X
1 f
8 s
1756 3605(2)N
11 s
1795 3588(,)N
2 f
1824(...)X
1 f
1917(is)X
2003(a)X
2069(sequence)X
2418(of)X
2518(positive)X
2825(integers)X
3131(such)X
3319(that)X
16 s
9 f
3479 3614(S)N
2 f
8 s
3555 3632(n)N
1 f
11 s
3594 3588(2)N
2 f
8 s
9 f
3540(-)Y
2 f
3683(l)X
5 s
3701 3552(n)N
1 f
11 s
9 f
3747 3588(\243)N
1 f
3809(1)X
3880(then)X
576 3774(there)N
780(is)X
867(a)X
934(pre\256x-code)X
2 f
1362(c)X
1 f
1415(:)X
2 f
1454(N)X
1 f
9 f
1534 MX
(->)174 987 oc
2 f
1635({)X
1 f
1670(0,)X
1757(1)X
2 f
(}*)S
1 f
1908(\(i.e.,)X
2095(if)X
2 f
2177(n)X
1 f
9 f
2242(\271)X
2 f
2304(m)X
1 f
2395(are)X
2530(positive)X
2838(integers,)X
3167(then)X
2 f
3346(c)X
1 f
3399(\()X
2 f
3428(n)X
1 f
3479(\))X
3535(is)X
3621(not)X
3761(a)X
3827(pre\256x)X
576 3934(of)N
2 f
686(c)X
1 f
739(\()X
2 f
768(m)X
1 f
838(\)\),)X
955(with)X
2 f
1150(l)X
1 f
1189(\()X
2 f
1218(c)X
1 f
1271(\()X
2 f
1300(n)X
1 f
1351(\)\))X
2 f
9 f
1423(=)X
2 f
1485(l)X
8 s
1510 3951(n)N
1 f
11 s
1549 3934(.)N
1631(Conversely,)X
2092(if)X
2 f
2184(c)X
1 f
2237(:)X
2 f
2276(N)X
1 f
9 f
2356 MX
(->)174 987 oc
2 f
2457({)X
1 f
2492(0,)X
2579(1)X
2 f
(}*)S
1 f
2740(is)X
2837(a)X
2914(pre\256x-code,)X
3374(then)X
3564(the)X
3710(sequence)X
2 f
576 4094(l)N
1 f
8 s
611 4111(1)N
11 s
650 4094(,)N
2 f
693(l)X
1 f
8 s
728 4111(2)N
11 s
767 4094(,)N
2 f
796(...)X
1 f
884(with)X
2 f
1063(l)X
8 s
1088 4111(n)N
11 s
9 f
1141 4094(=)N
2 f
1203(l)X
1 f
1242(\()X
2 f
1271(c)X
1 f
1324(\()X
2 f
1353(n)X
1 f
1404(\)\),)X
2 f
1506(n)X
9 f
1571(=)X
1 f
1633(1,)X
1720(2,)X
1793(...)X
1881(satis\256es)X
2182(the)X
2312(inequality)X
2688(above.)X
2942(See)X
3091(e.g.)X
3262([)X
3313(Gallager)X
776 4282(Hence,)N
1047(by)X
1160(the)X
1293(Kraft)X
1503(Inequality,)X
1908(for)X
2035(the)X
2168(pre\256x-code)X
2594(formed)X
2874(by)X
2988(the)X
3122(programs)X
2 f
3480(p)X
1 f
3550(of)X
2 f
3649(U)X
1 f
3738(we)X
3866(have)X
16 s
9 f
576 4468(S)N
2 f
8 s
652 4486(p)N
1 f
11 s
691 4442(2)N
2 f
8 s
9 f
4407(-)Y
2 f
770(l)X
1 f
798(\()X
2 f
819(p)X
1 f
856(\))X
11 s
9 f
898 4442(\243)N
1 f
960(1.)X
1071(Therefore,)X
1462(the)X
1593(combined)X
1964(probability)X
16 s
9 f
2374 4468(S)N
2 f
8 s
2450 4486(x)N
11 s
2485 4442(P)N
8 s
2539 4459(U)N
1 f
11 s
2592 4442(\()N
2 f
2621(x)X
1 f
2667(\),)X
2740(summed)X
3061(over)X
3239(all)X
2 f
3350(x)X
1 f
3389('s,)X
3496(sums)X
3699(up)X
3809(to)X
3900(less)X
576 4628(than)N
750(one,)X
921(no)X
1032(matter)X
1281(how)X
1455(we)X
1580(choose)X
1847(reference)X
2 f
2196(U)X
1 f
2259(,)X
2304(because)X
2605(for)X
2730(some)X
2939(program)X
2 f
3260(q)X
1 f
3327(there)X
3526(is)X
3608(no)X
3719(output)X
3968(at)X
576 4788(all.)N
776 4976(Another)N
1099(way)X
1280(to)X
1384(conceive)X
1732(of)X
2 f
1840(P)X
8 s
1894 4993(U)N
1 f
11 s
1947 4976(\()N
2 f
1976(x)X
1 f
2022(\))X
2086(is)X
2180(as)X
2288(follows.)X
2610(We)X
2768(think)X
2986(of)X
3095(the)X
3239(input)X
3457(to)X
3562(the)X
3706(reference)X
576 5136(pre\256x)N
805(machine)X
2 f
1128(U)X
1 f
1215(as)X
1312(being)X
1532(provided)X
1869(by)X
1981(inde\256nite)X
2339(long)X
2520(sequences)X
2899(of)X
2995(fair)X
3140(coin)X
3315(\257ips.)X
3534(Thus,)X
3755(the)X
3886(pro-)X
576 5296(bability)N
875(of)X
976(generating)X
1376(a)X
1443(program)X
2 f
1769(p)X
1 f
1841(for)X
2 f
1971(U)X
1 f
2063(is)X
2 f
2151(P)X
1 f
2219(\()X
2 f
2248(p)X
1 f
2299(\))X
2 f
9 f
2342(=)X
1 f
2404(2)X
2 f
8 s
9 f
5261(-)Y
2 f
2483(l)X
1 f
2511(\()X
2 f
2532(p)X
1 f
2569(\))X
11 s
2626 5296(where)N
2 f
2869(P)X
1 f
2952(is)X
3040(the)X
3177(standard)X
3504(`coin-\257ip')X
3890(uni-)X
576 5456(form)N
771(measure.)X
1132(\(Namely,)X
1486(presented)X
1847(with)X
2027(any)X
2177(in\256nitely)X
2520(long)X
2700(sequence)X
3045(starting)X
3333(with)X
2 f
3513(p)X
1 f
(,)S
3602(the)X
3733(machine)X
2 f
576 5616(U)N
1 f
639(,)X
690(being)X
916(a)X
985(pre\256x-machine,)X
1570(will)X
1738(read)X
1919(exactly)X
2 f
2204(p)X
1 f
2278(and)X
2435(no)X
2553(further.\))X
2895(Due)X
3071(to)X
3170(the)X
3308(halting)X
3584(problem,)X
3930(for)X
576 5776(some)N
2 f
784(q)X
1 f
850(the)X
980(reference)X
2 f
1328(U)X
1 f
1413(does)X
1596(not)X
1731(halt.)X
1908(Therefore,)X
2298(the)X
2 f
2428(halting)X
2701(probability)X
1 f
9 f
3116(W)X
1 f
3206(satis\256es)X
ep
%%Page: 18 18
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(18)X
2370(-)X
9 f
1971 736(W)N
2 f
9 f
2061(=)X
1 f
16 s
9 f
2131 762(S)N
2 f
8 s
2207 780(x)N
11 s
2256 736(P)N
8 s
2310 753(U)N
1 f
11 s
2363 736(\()N
2 f
2392(x)X
1 f
2438(\))X
2 f
2489(<)X
1 f
2570(1.)X
776 1030(Now)N
979(we)X
1115(are)X
1256(ready)X
1485(to)X
1588(state)X
1784(the)X
1926(remarkable)X
2356(and)X
2517(powerful)X
2868(fact)X
3034(that)X
3201(Levin's)X
3504(universal)X
3861(sem-)X
576 1190(icomputable)N
1044(measure)X
3 f
1368(m)X
1 f
1441(\()X
2 f
1470(x)X
1 f
1516(\),)X
1598(the)X
1737(Solomonoff-Levin)X
2423(universal)X
2 f
2776(a)X
2850(priori)X
1 f
3086(probability)X
2 f
3504(P)X
8 s
3558 1207(U)N
1 f
11 s
3611 1190(\()N
2 f
3640(x)X
1 f
3686(\),)X
3767(and)X
3924(the)X
576 1350(simpler)N
866(expression)X
1267(2)X
2 f
8 s
9 f
1315(-)Y
2 f
1346(K)X
1 f
1399(\()X
2 f
1420(x)X
1 f
1453(\))X
11 s
1481 1350(,)N
1528(all)X
1643(coincide)X
1968(up)X
2082(to)X
2177(an)X
2286(independent)X
2743(\256xed)X
2945(multipli)X
3227(cative)X
3464(constant.)X
3828(It)X
3908(is)X
3993(a)X
576 1510(consequence)N
1048(of)X
1144(universally)X
1559(accepted)X
1890(views)X
2118(in)X
2210(mathematical)X
2710(logic)X
2910(\(Church's)X
3283(Thesis\),)X
3586(that)X
3741(the)X
3871(wid-)X
576 1670(est)N
696(possible)X
1007(effective)X
1337(notion)X
1585(of)X
1680(simplicit)X
1991(y)X
2057(of)X
2152(description)X
2566(of)X
2661(an)X
2766(object)X
2 f
3004(x)X
1 f
3065(is)X
3146(quanti\256ed)X
3521(by)X
2 f
3631(K)X
1 f
3704(\()X
2 f
3733(x)X
1 f
3779(\).)X
776 1858(The)N
935(Solomonoff-Levin)X
1613(distribution)X
2043(can)X
2187(be)X
2292(interpreted)X
2697(as)X
2793(an)X
2899(recursively)X
3313(invariant)X
3650(notion)X
3899(that)X
576 2018(is)N
665(the)X
803(formal)X
1068(representation)X
1597(of)X
1700(``Occam's)X
2100(Razor'':)X
2423(the)X
2561(statement)X
2930(that)X
3093(one)X
3250(object)X
3496(is)X
3585(simpler)X
3880(than)X
576 2178(the)N
706(other)X
909(is)X
990(equivalent)X
1380(to)X
1471(saying)X
1723(that)X
1878(the)X
2008(former)X
2269(object)X
2507(has)X
2646(higher)X
2893(probability)X
3303(than)X
3477(the)X
3607(latter.)X
3 f
776 2366(Theorem)N
1144(3.)X
2 f
1259(There)X
1491(is)X
1577(a)X
1648(constant)X
1974(c)X
2040(such)X
2228(that)X
2393(for)X
2523(each)X
2716(x,)X
2804(up)X
2919(to)X
3015(additive)X
3327(constant)X
3653(c,)X
3741(we)X
3866(have)X
9 f
576 2526(-)N
1 f
638(log)X
3 f
765(m)X
1 f
838(\()X
2 f
867(x)X
1 f
913(\))X
2 f
9 f
956(=)X
1018(-)X
1 f
1080(log)X
2 f
1207(P)X
8 s
1261 2543(U)N
1 f
11 s
1314 2526(\()N
2 f
1343(x)X
1 f
1389(\))X
2 f
9 f
1432(=)X
2 f
1494(K)X
1 f
1567(\()X
2 f
1596(x)X
1 f
1642(\))X
2 f
1671(.)X
776 2714(Proof.)N
1 f
1064(Since)X
1303(2)X
2 f
8 s
9 f
2679(-)Y
2 f
1382(K)X
1 f
1435(\()X
2 f
1456(x)X
1 f
1489(\))X
11 s
1560 2714(represents)N
1959(the)X
2110(contribution)X
2585(to)X
2 f
2697(P)X
8 s
2751 2731(U)N
1 f
11 s
2804 2714(\()N
2 f
2833(x)X
1 f
2879(\))X
2951(by)X
3082(a)X
3165(shortest)X
3483(program)X
3825(for)X
2 f
3971(x)X
1 f
4010(,)X
576 2874(2)N
2 f
8 s
9 f
2839(-)Y
2 f
655(K)X
1 f
708(\()X
2 f
729(x)X
1 f
762(\))X
11 s
9 f
804 2874(\243)N
2 f
866(P)X
8 s
920 2891(U)N
1 f
11 s
973 2874(\()N
2 f
1002(x)X
1 f
1048(\))X
1101(for)X
1226(all)X
2 f
1338(x)X
1 f
1377(.)X
1444(Since)X
2 f
1663(P)X
8 s
1717 2891(U)N
1 f
11 s
1770 2874(\()N
2 f
1799(x)X
1 f
1845(\))X
1897(is)X
1979(semicomputable)X
2581(from)X
2775(below)X
3013(by)X
3124(enumerating)X
3588(all)X
3700(programs)X
576 3034(for)N
2 f
707(x)X
1 f
746(,)X
797(we)X
928(have)X
1123(by)X
1240(the)X
1377(universality)X
1823(of)X
3 f
1925(m)X
1 f
1998(\()X
2 f
2027(x)X
1 f
2073(\))X
2131(that)X
2293(there)X
2499(is)X
2588(a)X
2657(\256xed)X
2863(constant)X
2 f
3187(c)X
1 f
3256(such)X
3447(that)X
3610(for)X
3742(all)X
2 f
3861(x)X
1 f
3930(we)X
576 3194(have)N
2 f
764(P)X
8 s
818 3211(U)N
1 f
11 s
871 3194(\()N
2 f
900(x)X
1 f
946(\))X
9 f
989(\243)X
2 f
1051(c)X
3 f
1111(m)X
1 f
1184(\()X
2 f
1213(x)X
1 f
1259(\).)X
776 3382(It)N
855(remains)X
1159(to)X
1254(show)X
1465(that)X
3 f
1624(m)X
1 f
1697(\()X
2 f
1726(x)X
1 f
1772(\))X
2 f
9 f
1815(=)X
2 f
1877(O)X
1 f
1954(\(2)X
2 f
8 s
9 f
2027 3347(-)N
2 f
2062(K)X
1 f
2115(\()X
2 f
2136(x)X
1 f
2169(\))X
11 s
2197 3382(\).)N
2296(This)X
2479(is)X
2564(equivalent)X
2958(to)X
3053(showing)X
3377(that)X
3536(for)X
3664(some)X
3876(con-)X
576 3542(stant)N
2 f
768(c)X
1 f
832(we)X
959(have)X
2 f
9 f
1150(-)X
1 f
1212(log)X
3 f
1339(m)X
1 f
1412(\()X
2 f
1441(x)X
1 f
1487(\))X
9 f
1530(\263)X
2 f
1592(K)X
1 f
1665(\()X
2 f
1694(x)X
1 f
1740(\))X
2 f
9 f
1783(+)X
2 f
1845(c)X
1 f
1884(.)X
1953(It)X
2032(suf\256ces)X
2325(to)X
2419(exhibit)X
2690(a)X
2754(pre\256x)X
2983(code)X
3173(such)X
3358(that)X
3515(for)X
3641(some)X
3851(other)X
576 3702(\256xed)N
775(constant)X
2 f
1092(c)X
1 f
9 f
1138(\242)X
1 f
1160(,)X
1205(for)X
1330(each)X
2 f
1514(x)X
1 f
1576(there)X
1775(is)X
1857(a)X
1919(code)X
2108(word)X
2 f
2311(p)X
1 f
2378(such)X
2562(that)X
2 f
2719(l)X
1 f
2758(\()X
2 f
2787(p)X
1 f
2838(\))X
9 f
2881(\243)X
2 f
9 f
2943(-)X
1 f
3005(log)X
3 f
3132(m)X
1 f
3205(\()X
2 f
3234(x)X
1 f
3280(\))X
2 f
9 f
3323(+)X
2 f
3385(c)X
1 f
9 f
3431(\242)X
1 f
3453(,)X
3499(together)X
3812(with)X
3993(a)X
576 3862(pre\256x-machine)N
2 f
1138(T)X
1 f
1216(such)X
1406(that)X
2 f
1568(T)X
1 f
1631(\()X
2 f
1660(p)X
1 f
1711(\))X
2 f
9 f
1754(=)X
2 f
1816(x)X
1 f
1855(.)X
1906(Then,)X
2 f
2138(K)X
8 s
2197 3879(T)N
1 f
11 s
2240 3862(\()N
2 f
2269(x)X
1 f
2315(\))X
9 f
2358(\243)X
2 f
2420(l)X
1 f
2459(\()X
2 f
2488(p)X
1 f
2539(\))X
2597(and)X
2753(hence)X
2987(by)X
3104(the)X
3241(Invariance)X
3641(Theorem)X
3988(1)X
576 4022(also)N
2 f
747(K)X
1 f
820(\()X
2 f
849(x)X
1 f
895(\))X
9 f
938(\243)X
2 f
1000(l)X
1 f
1039(\()X
2 f
1068(p)X
1 f
1119(\))X
1177(up)X
1294(to)X
1392(a)X
1460(\256xed)X
1665(additive)X
1979(constant.)X
2346(First)X
2537(we)X
2668(recall)X
2893(a)X
2961(construction)X
3426(for)X
3557(the)X
3695(Shannon-)X
576 4182(Fano)N
774(code.)X
2 f
776 4370(Claim.)N
1 f
1072(If)X
9 f
1166(m)X
1 f
1253(is)X
1348(a)X
1423(measure)X
1752(on)X
1876(the)X
2020(integers,)X
16 s
9 f
2357 4396(S)N
2 f
8 s
2433 4414(x)N
1 f
11 s
9 f
2468 4370(m)N
1 f
2519(\()X
2 f
2548(x)X
1 f
2594(\))X
9 f
2637(\243)X
1 f
2699(1,)X
2801(then)X
2989(there)X
3201(is)X
3296(a)X
3371(binary)X
3632(pre\256x-code)X
2 f
576 4556(r)N
1 f
624(:)X
2 f
663(N)X
1 f
9 f
743 MX
(->)174 987 oc
2 f
844({)X
1 f
879(0,)X
966(1)X
2 f
(}*)S
1 f
1131(such)X
1334(that)X
1509(the)X
1659(code)X
1867(words)X
2 f
2123(r)X
1 f
2171(\(1\),)X
2 f
2316(r)X
1 f
2364(\(2\),)X
2495(...)X
2603(are)X
2752(in)X
2863(lexicographical)X
3449(order,)X
3697(such)X
3899(that)X
2 f
576 4716(l)N
1 f
615(\()X
2 f
644(r)X
1 f
692(\()X
2 f
721(x)X
1 f
767(\)\))X
9 f
839(\243)X
2 f
9 f
901(-)X
1 f
963(log)X
9 f
1090(m)X
1 f
1141(\()X
2 f
1170(x)X
1 f
1216(\))X
2 f
9 f
1259(+)X
1 f
1321(2.)X
1409(This)X
1588(is)X
1669(the)X
1799(Shannon-Fano)X
2334(code.)X
2 f
776 4904(Proof.)N
1 f
1043(Let)X
1183([0,)X
1299(1\))X
1394(be)X
1499(the)X
1629(half)X
1788(open)X
1982(unit)X
2143(real)X
2298(interval.)X
2635(The)X
2795(half)X
2955(open)X
3149(interval)X
3442([0.)X
2 f
3537(x)X
1 f
3576(,)X
3626(0.)X
2 f
(x)S
9 f
3752(+)X
1 f
3814(2)X
2 f
8 s
9 f
4869(-)Y
2 f
3893(l)X
1 f
3921(\()X
2 f
3942(x)X
1 f
3975(\))X
11 s
4003 4904(\))N
576 5064(corresponding)N
1108(to)X
1206(the)X
1343(set)X
1470(\(cylinder\))X
1845(of)X
1946(reals)X
9 f
2140(G)X
2 f
8 s
2193 5081(x)N
11 s
9 f
2242 5064(=)N
2 f
2304({)X
1 f
2339(0.)X
2 f
(y)S
1 f
2458(:)X
2 f
2497(y)X
9 f
2557(=)X
2 f
2619(x)X
2679(z)X
2720(})X
1 f
2783(\()X
2 f
2812(x)X
1 f
2879(\256nite)X
3089(and)X
2 f
3244(y)X
1 f
3311(and)X
2 f
3466(z)X
1 f
3528(in\256nite)X
3807(binary)X
576 5224(strings\))N
874(is)X
967(called)X
1212(a)X
2 f
1285(binary)X
1549(interval)X
1 f
1824(.)X
1902(We)X
2059(cut)X
2202(off)X
2339(disjoint,)X
2662(consecutive,)X
3135(adjacent)X
3464(\(not)X
3641(necessarily)X
576 5384(binary\))N
852(intervals)X
2 f
1178(I)X
8 s
1207 5401(n)N
1 f
11 s
1268 5384(of)N
1363(length)X
9 f
1606(m)X
1 f
1657(\()X
2 f
1686(n)X
1 f
1737(\))X
1788(from)X
1981(the)X
2111(left)X
2251(end)X
2400(of)X
2495([0,)X
2611(1\),)X
2 f
2728(n)X
9 f
2793(=)X
1 f
2855(1,)X
2942(2,)X
3015(....)X
3125(Let)X
2 f
3265(i)X
8 s
3290 5401(n)N
1 f
11 s
3351 5384(be)N
3456(the)X
3586(length)X
3829(of)X
3924(the)X
576 5544(longest)N
863(binary)X
1120(interval)X
1422(contained)X
1797(in)X
2 f
1898(I)X
8 s
1927 5561(n)N
1 f
11 s
1966 5544(.)N
2020(Set)X
2 f
2165(r)X
1 f
2213(\()X
2 f
2242(n)X
1 f
2293(\))X
2354(equal)X
2577(to)X
2678(the)X
2818(binary)X
3075(word)X
3287(corresponding)X
3822(to)X
3924(the)X
576 5704(\256rst)N
737(such)X
922(interval.)X
1260(It)X
1338(is)X
1421(easy)X
1601(to)X
1694(see)X
1830(that)X
2 f
1987(I)X
8 s
2016 5721(n)N
1 f
11 s
2079 5704(is)N
2162(covered)X
2464(by)X
2576(at)X
2664(most)X
2860(four)X
3030(binary)X
3279(intervals)X
3606(of)X
3702(length)X
2 f
3946(i)X
8 s
3971 5721(n)N
1 f
11 s
4010 5704(,)N
576 5864(from)N
769(which)X
1006(the)X
1136(claim)X
1355(follows.)X
10 f
1663(`)X
ep
%%Page: 19 19
bp
11 s 0 xH 0 xS 10 f
1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(19)X
2370(-)X
776 736(Since)N
3 f
997(m)X
1 f
1070(\()X
2 f
1099(x)X
1 f
1145(\))X
1199(is)X
1283(semicomputable)X
1888(from)X
2085(below,)X
2348(there)X
2550(is)X
2635(a)X
2700(partial)X
2952(recursive)X
3300(function)X
9 f
3620(f)X
1 f
3666(\()X
2 f
3695(t)X
1 f
3720(,)X
2 f
3770(x)X
1 f
3816(\))X
3871(such)X
576 896(that)N
9 f
751(f)X
1 f
797(\()X
2 f
826(t)X
1 f
851(,)X
2 f
901(x)X
1 f
947(\))X
9 f
990(\243)X
3 f
1052(m)X
1 f
1125(\()X
2 f
1154(x)X
1 f
1200(\))X
1271(for)X
1415(all)X
2 f
1546(t)X
1 f
1571(,)X
1634(and)X
1802(lim)X
2 f
8 s
1921 913(t)N
1 f
9 f
1954 MX
(->)174 987 oc
2027(\245)X
11 s
2080 896(f)N
1 f
2126(\()X
2 f
2155(t)X
1 f
2180(,)X
2 f
2230(x)X
1 f
2276(\))X
2 f
9 f
2319(=)X
3 f
2381(m)X
1 f
2454(\()X
2 f
2483(x)X
1 f
2529(\).)X
2643(Let)X
9 f
2802(y)X
1 f
2862(\()X
2 f
2891(t)X
1 f
2916(,)X
2 f
2966(x)X
1 f
3012(\))X
2 f
9 f
3055(=)X
1 f
3117(2)X
2 f
8 s
9 f
861(-)Y
2 f
3196(k)X
1 f
11 s
3231 896(,)N
3294(with)X
2 f
3492(k)X
1 f
3572(is)X
3672(a)X
3752(positive)X
576 1056(integer,)N
867(be)X
974(the)X
1106(greatest)X
1404(partial)X
1654(recursive)X
2000(lower)X
2224(bound)X
2468(of)X
2565(this)X
2718(form)X
2914(on)X
9 f
3027(f)X
1 f
3073(\()X
2 f
3102(t)X
1 f
3127(,)X
2 f
3177(x)X
1 f
3223(\).)X
3321(We)X
3468(can)X
3615(assume)X
3899(that)X
9 f
576 1216(y)N
1 f
658(enumerates)X
1081(its)X
1187(range)X
1404(without)X
1696(repetition.)X
2079(Then,)X
16 s
9 f
1525 1482(S)N
2 f
8 s
1601 1500(x)N
1 f
1629(,)X
2 f
1665(t)X
1 f
11 s
9 f
1690 1456(y)N
1 f
1750(\()X
2 f
1779(t)X
1 f
1804(,)X
2 f
1854(x)X
1 f
1900(\))X
2 f
9 f
1951(=)X
1 f
16 s
9 f
2021 1482(S)N
2 f
8 s
2097 1500(x)N
1 f
16 s
9 f
2146 1482(S)N
2 f
8 s
2222 1500(t)N
1 f
11 s
9 f
2247 1456(y)N
1 f
2307(\()X
2 f
2336(t)X
1 f
2361(,)X
2 f
2411(x)X
1 f
2457(\))X
9 f
2508(\243)X
16 s
2578 1482(S)N
2 f
8 s
2654 1500(x)N
1 f
11 s
2689 1456(2)N
3 f
2747(m)X
1 f
2820(\()X
2 f
2849(x)X
1 f
2895(\))X
9 f
2946(\243)X
1 f
3016(2.)X
576 1722(\(The)N
771(series)X
16 s
9 f
1000 1748(S)N
2 f
8 s
1076 1766(t)N
1 f
11 s
9 f
1101 1722(y)N
1 f
1161(\()X
2 f
1190(t)X
1 f
1215(,)X
2 f
1265(x)X
1 f
1311(\))X
1369(can)X
1520(only)X
1706(converge)X
2057(to)X
2155(precisely)X
2502(2)X
3 f
2560(m)X
1 f
2633(\()X
2 f
2662(x)X
1 f
2708(\))X
2766(in)X
2864(case)X
3044(there)X
3250(is)X
3339(a)X
3408(positive)X
3718(integer)X
2 f
3993(k)X
1 f
576 1908(such)N
759(that)X
3 f
914(m)X
1 f
987(\()X
2 f
1016(x)X
1 f
1062(\))X
2 f
9 f
1105(=)X
1 f
1167(2)X
2 f
8 s
9 f
1873(-)Y
2 f
1246(k)X
1 f
11 s
1281 1908(.\))N
776 2096(Similar)N
1075(to)X
1182(before,)X
1466(we)X
1606(chop)X
1815(off)X
1955(consecutive,)X
2431(adjacent,)X
2785(disjoint)X
3089(half)X
3264(open)X
3474(intervals)X
2 f
3817(I)X
8 s
3846 2113(t)N
1 f
3864(,)X
2 f
3885(x)X
1 f
11 s
3959 2096(of)N
576 2256(length)N
9 f
828(y)X
1 f
888(\()X
2 f
917(t)X
1 f
942(,)X
2 f
992(x)X
1 f
1038(\))X
2 f
1074(/)X
1 f
1106(2,)X
1203(in)X
1303(order)X
1519(of)X
1622(computation)X
2094(of)X
9 f
2197(y)X
1 f
2257(\()X
2 f
2286(t)X
1 f
2311(,)X
2 f
2361(x)X
1 f
2407(\),)X
2488(starting)X
2783(from)X
2984(the)X
3122(left)X
3270(side)X
3442(of)X
3545([0,)X
3661(1\).)X
3786(This)X
3973(is)X
576 2416(possible)N
889(by)X
1001(the)X
1133(last)X
1280(displayed)X
1642(equation.)X
2014(It)X
2092(is)X
2175(easy)X
2355(to)X
2448(see)X
2584(that)X
2741(we)X
2867(can)X
3013(construct)X
3361(a)X
3425(pre\256x-machine)X
2 f
3983(T)X
1 f
576 2576(as)N
676(follows.)X
989(If)X
9 f
1074(G)X
2 f
8 s
1127 2593(p)N
1 f
11 s
1193 2576(is)N
1279(the)X
1414(largest)X
1676(binary)X
1928(interval)X
2225(of)X
2 f
2325(I)X
8 s
2354 2593(t)N
1 f
2372(,)X
2 f
2393(x)X
1 f
11 s
2428 2576(,)N
2477(then)X
2 f
2656(T)X
1 f
2719(\()X
2 f
2748(p)X
1 f
2799(\))X
2 f
9 f
2842(=)X
2 f
2904(x)X
1 f
2943(.)X
2992(Otherwise,)X
2 f
3402(T)X
1 f
3465(\()X
2 f
3494(p)X
1 f
3545(\))X
3600(is)X
3685(unde\256ned)X
576 2736(\(e.g.,)N
2 f
776(T)X
1 f
847(doesn't)X
1128(halt\).)X
776 2924(By)N
915(construction)X
1387(of)X
9 f
1496(y)X
1 f
1556(,)X
1614(for)X
1752(each)X
2 f
1949(x)X
1 f
2024(there)X
2236(is)X
2331(a)X
9 f
2407(y)X
1 f
2467(\()X
2 f
2496(t)X
1 f
2521(,)X
2 f
2571(x)X
1 f
2617(\))X
2 f
2660(>)X
3 f
2719(m)X
1 f
2792(\()X
2 f
2821(x)X
1 f
2867(\))X
2 f
2903(/)X
1 f
2935(2.)X
3060(By)X
3200(the)X
3345(construction)X
3818(in)X
3924(the)X
576 3084(Claim,)N
851(each)X
1048(interval)X
2 f
1354(I)X
8 s
1383 3101(t)N
1 f
1401(,)X
2 f
1422(x)X
1 f
11 s
1493 3084(has)N
1646(length)X
9 f
1903(y)X
1 f
1963(\()X
2 f
1992(t)X
1 f
2017(,)X
2 f
2067(x)X
1 f
2113(\))X
2 f
2149(/)X
1 f
2181(2.)X
2283(Each)X
2 f
2495(I)X
1 f
2524(-interval)X
2859(contains)X
3189(a)X
3264(binary)X
3525(interval)X
9 f
3831(G)X
2 f
8 s
3884 3101(p)N
1 f
11 s
3959 3084(of)N
576 3244(length)N
848(at)X
963(least)X
1176(one)X
1354(quarter)X
1654(of)X
1779(that)X
1964(of)X
2 f
2089(I)X
1 f
2118(.)X
2192(Therefore,)X
2612(there)X
2840(is)X
2951(a)X
2 f
3042(p)X
1 f
3138(with)X
2 f
3347(T)X
1 f
3410(\()X
2 f
3439(p)X
1 f
3490(\))X
2 f
9 f
3533(=)X
2 f
3595(x)X
1 f
3686(such)X
3899(that)X
576 3404(2)N
2 f
8 s
9 f
3369(-)Y
2 f
655(l)X
1 f
683(\()X
2 f
704(p)X
1 f
741(\))X
11 s
9 f
783 3404(\263)N
3 f
845(m)X
1 f
918(\()X
2 f
947(x)X
1 f
993(\))X
2 f
1029(/)X
1 f
1061(16.)X
1215(This)X
1394(implies)X
2 f
1677(K)X
8 s
1736 3421(T)N
1 f
11 s
1779 3404(\()N
2 f
1808(x)X
1 f
1854(\))X
9 f
1897(\243)X
2 f
9 f
1959(-)X
1 f
2007(log)X
3 f
2134(m)X
1 f
2207(\()X
2 f
2236(x)X
1 f
2282(\))X
2 f
9 f
2325(+)X
1 f
2387(4.)X
2475(The)X
2634(proof)X
2846(of)X
2941(the)X
3071(theorem)X
3382(is)X
3463(\256nished.)X
10 f
3808(`)X
1 f
776 3592(Theorem)N
1117(3)X
1184(demonstrates)X
1672(a)X
1735(particularly)X
2166(important)X
2534(instance)X
2847(of)X
2944(the)X
3076(two)X
3232(conceptually)X
3707(different,)X
576 3752(but)N
730(equivalent,)X
1161(de\256nitions)X
1575(of)X
1689(the)X
1838(semicomputable)X
2457(measures.)X
2868(We)X
3030(analyse)X
3334(this)X
3502(equivalence)X
3963(in)X
576 3912(some)N
785(detail.)X
1050(Let)X
2 f
1192(P)X
1 f
8 s
1256 3929(1)N
11 s
1295 3912(,)N
2 f
1338(P)X
1 f
8 s
1402 3929(2)N
11 s
1441 3912(,)N
2 f
1470(...)X
1 f
1560(be)X
1667(the)X
1799(effective)X
2131(enumeration)X
2596(of)X
2693(all)X
2806(semicomputable)X
3409(probability)X
3821(distri-)X
576 4072(butions)N
861(constructed)X
1292(in)X
1386(Theorem)X
1729(2.)X
1820(Let)X
2 f
1962(T)X
1 f
8 s
2021 4089(1)N
11 s
2060 4072(,)N
2 f
2103(T)X
1 f
8 s
2162 4089(2)N
11 s
2201 4072(,)N
2 f
2230(...)X
1 f
2320(be)X
2427(the)X
2559(standard)X
2881(enumeration)X
3346(of)X
3443(pre\256x-machines.)X
576 4232(For)N
720(each)X
903(pre\256x-machine)X
2 f
1458(T)X
1 f
1507(,)X
1551(de\256ne)X
2 f
1903 4472(Q)N
8 s
1966 4489(T)N
1 f
11 s
2009 4472(\()N
2 f
2038(x)X
1 f
2084(\))X
2 f
9 f
2135(=)X
1 f
16 s
9 f
2205 4498(S)N
2 f
8 s
2281 4516(T)N
1 f
2327(\()X
2 f
2348(p)X
1 f
2385(\))X
2 f
9 f
2416(=)X
2 f
2461(x)X
1 f
11 s
2496 4472(2)N
2 f
8 s
9 f
4437(-)Y
2 f
2575(l)X
1 f
2603(\()X
2 f
2624(p)X
1 f
2661(\).)X
11 s
576 4738(In)N
680(other)X
892(words,)X
2 f
1159(Q)X
8 s
1222 4755(T)N
1 f
11 s
1265 4738(\()N
2 f
1294(x)X
1 f
1340(\))X
1400(is)X
1490(the)X
1629(probability)X
2048(that)X
2 f
2212(T)X
1 f
2292(computes)X
2661(output)X
2 f
2918(x)X
1 f
2988(if)X
3073(its)X
3189(input)X
2 f
3403(p)X
1 f
3479(is)X
3570(generated)X
3944(by)X
576 4898(successive)N
979(tosses)X
1221(of)X
1326(a)X
1397(fair)X
1551(coin.)X
1757(In)X
1862(other)X
2075(words,)X
2343(the)X
2483(inputs)X
2 f
2730(p)X
1 f
2805(are)X
2943(uniformly)X
3327(distributed)X
3736(with)X
3924(the)X
576 5058(probability)N
986(of)X
2 f
1081(p)X
1 f
1147(occurring)X
1506(equal)X
1719(2)X
2 f
8 s
9 f
5023(-)Y
2 f
1798(l)X
1 f
1826(\()X
2 f
1847(p)X
1 f
1884(\))X
11 s
1912 5058(.)N
1956(It)X
2032(is)X
2113(easy)X
2291(to)X
2382(see)X
2516(that)X
2671(each)X
2 f
2854(Q)X
8 s
2917 5075(T)N
1 f
11 s
2982 5058(satis\256es)N
2 f
8 s
2099 5388(x)N
1 f
16 s
9 f
2075 5324(S)N
2 f
11 s
2165 5298(Q)N
8 s
2228 5315(T)N
1 f
11 s
2271 5298(\()N
2 f
2300(x)X
1 f
2346(\))X
9 f
2397(\243)X
1 f
2467(1.)X
576 5610(Equality)N
914(holds)X
1143(iff)X
2 f
1264(T)X
1 f
1351(halts)X
1556(for)X
1696(all)X
1823(inputs)X
2077(\(proper)X
2373(programs\).)X
2816(Let)X
2 f
2972(Q)X
1 f
8 s
3045 5627(1)N
11 s
3084 5610(,)N
2 f
3127(Q)X
1 f
8 s
3200 5627(2)N
11 s
3239 5610(,)N
2 f
3268(...)X
1 f
3372(\(where)X
3653(we)X
3793(do)X
3919(not)X
576 5770(require)N
847(equality)X
1154(to)X
1245(hold\))X
1453(be)X
1558(the)X
1688(probability)X
2098(distributions)X
2562(associated)X
2946(with)X
2 f
3125(T)X
1 f
8 s
3184 5787(1)N
11 s
3223 5770(,)N
2 f
3266(T)X
1 f
8 s
3325 5787(2)N
11 s
3364 5770(,)N
2 f
3393(...)X
1 f
(.)S
ep
%%Page: 20 20
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(20)X
2370(-)X
2 f
776 736(Claim.)N
1 f
1069(There)X
1307(are)X
1447(recursive)X
1802(functions)X
9 f
2163(s)X
1 f
2216(,)X
9 f
2259(p)X
1 f
2341(such)X
2536(that)X
2 f
2703(Q)X
8 s
2766 753(n)N
11 s
9 f
2819 736(=)N
1 f
9 f
2881(Q)X
1 f
2946(\()X
2 f
2975(P)X
1 f
8 s
9 f
3039 753(s)N
1 f
3078(\()X
2 f
3099(n)X
1 f
3136(\))X
11 s
3164 736(\))N
3227(and)X
2 f
3388(P)X
8 s
3442 753(n)N
11 s
9 f
3495 736(=)N
1 f
9 f
3557(Q)X
1 f
3622(\()X
2 f
3651(Q)X
1 f
8 s
9 f
3724 753(p)N
1 f
3759(\()X
2 f
3780(n)X
1 f
3817(\))X
11 s
3845 736(\),)N
3930(for)X
2 f
576 896(n)N
9 f
641(=)X
1 f
689(1,)X
776(2,)X
849(....)X
2 f
776 1084(Proof.)N
1 f
1043(Omitted.)X
10 f
1399(`)X
3 f
776 1272(Remark.)N
1 f
1187(The)X
1385(Coding)X
1706(Theorem)X
2085(tells)X
2294(us)X
2433(that)X
2627(there)X
2864(is)X
2984(a)X
3084(constant)X
2 f
3439(c)X
3499(>)X
1 f
3572(0)X
3677(such)X
3899(that)X
2 f
9 f
576 1432(-)N
1 f
638(log)X
2 f
765(P)X
8 s
819 1449(U)N
1 f
11 s
872 1432(\()N
2 f
901(x)X
1 f
947(\))X
2 f
9 f
990(-)X
2 f
1052(K)X
1 f
1125(\()X
2 f
1154(x)X
1 f
1200(\))X
9 f
1243(\243)X
2 f
1305(c)X
1 f
1344(.)X
1402(We)X
1560(recall)X
1791(from)X
1997(the)X
2140(de\256nition)X
2514(of)X
2622(the)X
2765(Solomonoff-Levin)X
3456(distribution)X
3899(that)X
2 f
9 f
576 1592(-)N
1 f
638(log)X
2 f
765(P)X
8 s
819 1609(U)N
1 f
11 s
872 1592(\()N
2 f
901(x)X
1 f
947(\))X
2 f
9 f
990(=)X
1052(-)X
1 f
1114(log)X
16 s
9 f
1241 1618(S)N
2 f
8 s
1317 1636(U)N
1 f
1373(\()X
2 f
1394(p)X
1 f
1431(\))X
2 f
9 f
1462(=)X
2 f
1507(x)X
1 f
11 s
1542 1592(2)N
2 f
8 s
9 f
1557(-)Y
2 f
1621(l)X
1 f
1649(\()X
2 f
1670(p)X
1 f
1707(\))X
11 s
1735 1592(,)N
1794(and)X
2 f
1958(K)X
1 f
2031(\()X
2 f
2060(x)X
1 f
2106(\))X
2 f
9 f
2149(=)X
1 f
2211(min)X
2 f
2363({l)X
1 f
2437(\()X
2 f
2466(p)X
1 f
2517(\):)X
2 f
2585(U)X
1 f
2662(\()X
2 f
2691(p)X
1 f
2742(\))X
2 f
9 f
2785(=)X
2 f
2847(x)X
2893(})X
1 f
2928(.)X
2 f
3009(A)X
3100(priori)X
1 f
3343(an)X
3463(outcome)X
2 f
3804(x)X
1 f
3880(may)X
576 1778(have)N
767(high)X
949(probability)X
1362(because)X
1665(it)X
1740(has)X
1882(many)X
2102(long)X
2283(descriptions.)X
2755(But)X
2907(these)X
3112(relations)X
3440(show)X
3649(that)X
3806(in)X
3899(that)X
576 1938(case)N
750(it)X
823(must)X
1018(have)X
1207(a)X
1269(short)X
1468(description)X
1883(too.)X
2063(In)X
2159(other)X
2363(words,)X
2622(the)X
2 f
2753(a)X
2820(priori)X
1 f
3050(probability)X
3462(of)X
2 f
3559(x)X
1 f
3622(is)X
3705(governed)X
576 2098(by)N
686(the)X
816(shortest)X
1112(program)X
1432(for)X
2 f
1556(x)X
1 f
1595(.)X
3 f
776 2286(Remark.)N
1 f
1173(Let)X
2 f
1337(P)X
1 f
1437(be)X
1566(any)X
1739(probability)X
2173(distribution)X
2628(\(not)X
2817(necessarily)X
3255(computable\).)X
3765(The)X
2 f
3949(P)X
1 f
4003(-)X
576 2446(expected)N
911(value)X
1124(of)X
3 f
1219(m)X
1 f
1292(\()X
2 f
1321(x)X
1 f
1367(\))X
2 f
1403(/)X
1435(P)X
1 f
1503(\()X
2 f
1532(x)X
1 f
1578(\))X
1629(is)X
16 s
9 f
1954 2747(S)N
2 f
8 s
2030 2765(x)N
11 s
2079 2721(P)N
1 f
2147(\()X
2 f
2176(x)X
1 f
2222(\))X
2 f
2289 2782(P)N
1 f
2357(\()X
2 f
2386(x)X
1 f
2432(\))X
3 f
2287 2669(m)N
1 f
2360(\()X
2 f
2389(x)X
1 f
2435(\))X
10 f
2273 2695(h)N
2301(hhhh)X
2 f
2507 2721(<)N
1 f
2588(1.)X
576 3005(We)N
720(\256nd)X
879(by)X
989(Chebychev's)X
1470(\256rst)X
1629(Inequality)X
1987 2961(1\))N
2082 3005(that)N
16 s
9 f
1722 3306(S)N
2 f
11 s
1798 3280({P)N
1 f
1901(\()X
2 f
1930(x)X
1 f
1976(\):)X
3 f
2044(m)X
1 f
2117(\()X
2 f
2146(x)X
1 f
2192(\))X
9 f
2235(\243)X
2 f
2297(k)X
2357(P)X
1 f
2425(\()X
2 f
2454(x)X
1 f
2500(\))X
2 f
2529(})X
1 f
9 f
2586(\263)X
1 f
2656(1)X
2 f
9 f
2714(-)X
2 f
2800 3341(k)N
1 f
2798 3228(1)N
10 f
2784 3254(h)N
2811(h)X
2 f
2863 3280(.)N
1 f
3930(\(3\))X
776 3592(Since)N
3 f
994(m)X
1 f
1067(\()X
2 f
1096(x)X
1 f
1142(\))X
1193(dominates)X
1578(all)X
1689(semicomputable)X
2290(measures)X
2639(multipli)X
2921(catively,)X
3245(for)X
3369(all)X
2 f
3480(x)X
1 f
3541(we)X
3665(have)X
2 f
2019 3832(P)N
1 f
2087(\()X
2 f
2116(x)X
1 f
2162(\))X
9 f
2213(\243)X
2 f
2283(c)X
8 s
2322 3849(P)N
3 f
11 s
2382 3832(m)N
1 f
2455(\()X
2 f
2484(x)X
1 f
2530(\),)X
3930(\(4\))X
576 4072(for)N
701(a)X
763(\256xed)X
962(positive)X
1265(constant)X
2 f
1582(c)X
8 s
1621 4089(P)N
1 f
11 s
1690 4072(independent)N
2144(of)X
2 f
2240(x)X
1 f
2303(\(but)X
2469(depending)X
2860(on)X
2972(the)X
3104(index)X
3324(of)X
2 f
3421(P)X
1 f
3499(in)X
3592(the)X
3724(effective)X
576 4232(enumeration)N
9 f
1039(m)X
8 s
1 f
1090 4249(1)N
11 s
1129 4232(,)N
9 f
1172(m)X
8 s
1 f
1223 4249(2)N
11 s
1262 4232(,)N
2 f
1291(...)X
1 f
1379(of)X
1474(semicomputable)X
2075(measures\).)X
776 4420(Inequalities)N
1210(\(3\))X
1334(and)X
1483(\(4\))X
1607(have)X
1795(the)X
1925(following)X
2290(consequences:)X
2798 4376(2\))N
776 4608(\(i\))N
882(If)X
2 f
963(x)X
1 f
1025(is)X
1107(a)X
1169(random)X
1461(sample)X
1734(from)X
1928(a)X
1990(simple)X
2249(computable)X
2684(distribution)X
2 f
3115(P)X
1 f
3183(\()X
2 f
3212(x)X
1 f
3258(\),)X
3333(then)X
3 f
3509(m)X
1 f
3582(\()X
2 f
3611(x)X
1 f
3657(\))X
3710(is)X
3793(a)X
3856(good)X
576 4768(estimate)N
893(of)X
2 f
988(P)X
1 f
1056(\()X
2 f
1085(x)X
1 f
1131(\).)X
776 4956(\(ii\))N
912(If)X
998(we)X
1128(know)X
1351(or)X
1452(believe)X
1735(that)X
2 f
1896(x)X
1 f
1963(is)X
2050(random)X
2348(with)X
2534(respect)X
2812(to)X
2 f
2910(P)X
1 f
2964(,)X
3015(and)X
3171(we)X
3302(know)X
2 f
3526(P)X
1 f
3594(\()X
2 f
3623(x)X
1 f
3669(\),)X
3749(then)X
3930(we)X
576 5116(can)N
720(use)X
2 f
859(P)X
1 f
927(\()X
2 f
956(x)X
1 f
1002(\))X
1053(as)X
1148(an)X
1253(estimate)X
1570(of)X
3 f
1665(m)X
1 f
1738(\()X
2 f
1767(x)X
1 f
1813(\).)X
9 s
10 f
576 5204(hhhhhhhhhhhhhhhh)N
1 f
576 5284(1\))N
654 5320(Recall)N
856(that)X
2 f
982(Chebychev's)X
1368(First)X
1526(Inequality)X
1 f
1836(says)X
1978(the)X
2085(following.)X
2420(Let)X
2 f
2535(P)X
1 f
2598(be)X
2685(any)X
2808(probability)X
3143(distribution,)X
2 f
3518(f)X
1 f
3557(any)X
3680(nonnegative)X
576 5408(function)N
843(with)X
998(expected)X
1281(value)X
2 f
1464(E)X
6 s
5422(P)Y
1 f
9 s
1543 5408(\()N
2 f
1573(f)X
1 f
1611(\))X
2 f
9 f
1647(=)X
1 f
14 s
9 f
1699 5429(S)N
2 f
6 s
1765 5444(x)N
9 s
1792 5408(P)N
1 f
1848(\()X
2 f
(x)S
1 f
1910(\))X
2 f
1952(f)X
1 f
1996(\()X
2 f
(x)S
1 f
2058(\))X
2 f
2094(<)X
1 f
9 f
2155(\245)X
1 f
2206(.)X
2269(For)X
9 f
2395(l)X
2447(\263)X
1 f
2499(0)X
2561(we)X
2671(have)X
14 s
9 f
2833 5429(S)N
2 f
9 s
2899 5408({P)N
1 f
2984(\()X
2 f
(x)S
1 f
3046(\):)X
2 f
3108(f)X
1 f
3152(\()X
2 f
(x)S
1 f
3214(\))X
2 f
3250(>)X
1 f
9 f
3311(l)X
2 f
3351(})X
3392(<)X
1 f
9 f
3453(l)X
2 f
6 s
9 f
3493 5380(-)N
1 f
3519(1)X
2 f
9 s
3549 5408(E)N
6 s
5422(P)Y
1 f
9 s
3628 5408(\()N
2 f
3658(f)X
1 f
3696(\).)X
3782(Here)X
3948(we)X
576 5517(use)N
690(it)X
748(with)X
2 f
894(k)X
944(E)X
6 s
5531(P)Y
1 f
9 s
1023 5517(\()N
2 f
1053(f)X
1 f
1091(\))X
1133(substituted)X
1463(for)X
9 f
1565(l)X
1 f
1605(.)X
576 5597(2\))N
656 5633(We)N
776(shortly)X
995(remark,)X
1236(without)X
1475(further)X
1690(explanation,)X
2063(that)X
2190(in)X
2265(both)X
2412(cases)X
2583(the)X
2690(degree)X
2901(of)X
2980(approximation)X
3419(depends)X
3674(on)X
3765(the)X
3872(index)X
576 5721(of)N
2 f
654(P)X
1 f
(,)S
734(and)X
856(the)X
962(randomness)X
1324(of)X
2 f
1402(x)X
1 f
1452(with)X
1598(respect)X
1820(to)X
2 f
1894(P)X
1 f
(,)S
1974(as)X
2052(measured)X
2346(by)X
2436(the)X
2542(randomness)X
2905(de\256ciency)X
9 f
3220(d)X
6 s
1 f
3256 5735(0)N
9 s
3286 5721(\()N
2 f
(x)S
1 f
9 f
3366(|)X
2 f
3404(P)X
1 f
3454(\))X
2 f
9 f
3490(=)X
1 f
3542(log)X
3646(\()X
3 f
(m)S
1 f
(\()S
2 f
(x)S
1 f
3792(\))X
2 f
3822(/)X
3848(P)X
1 f
3904(\()X
2 f
(x)S
1 f
3966(\)\).)X
576 5809(If)N
9 f
667(d)X
6 s
1 f
703 5823(0)N
9 s
733 5809(\()N
2 f
(x)S
1 f
9 f
813(|)X
2 f
851(P)X
1 f
901(\))X
2 f
9 f
937(=)X
2 f
989(O)X
1 f
1053(\(1\))X
1180(then)X
2 f
1347(x)X
1 f
1422(is)X
1512(random,)X
1792(otherwise)X
2 f
2114(x)X
1 f
2188(is)X
2278(not)X
2412(random.)X
2692(For)X
2834(example,)X
3138(for)X
3264(the)X
3394(Uniform)X
3684(Distribution)X
ep
%%Page: 21 21
bp
9 s 0 xH 0 xS 1 f
11 s
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(21)X
2370(-)X
3 f
576 736(2.4.1.)N
818(Solomonoff's)X
1333(Inference)X
1711(Procedure)X
2124(and)X
2288(Its)X
2407(Mathematical)X
2952(Justi\256cation)X
1 f
576 924(We)N
720(continue)X
1046(Solomonoff's)X
1553(approach)X
1898(to)X
1990(inductive)X
2342(inference,)X
2714(as)X
2810(in)X
2902([)X
2954(Levin)X
3183(Zvonkin)X
3505(In)X
3601(general,)X
3905(one)X
576 1084(cannot)N
845(prove)X
1079(that)X
1246(an)X
1362(inference)X
1722(procedure)X
2106(in)X
2208(statistics)X
2546(is)X
2638(good.)X
2869(This)X
3059(accounts)X
3400(for)X
3535(the)X
3676(many)X
3905(dif-)X
576 1244(ferent)N
803(approaches)X
1220(which)X
1458(are)X
1588(advocated)X
1968(in)X
2060(statistics.)X
2410(In)X
2506(contrast,)X
2830(about)X
3049(Solomonoff's)X
3556(procedure)X
3930(we)X
576 1404(can)N
723(rigourously)X
1154(prove)X
1379(that)X
1537(it)X
1612(is)X
1696(good.)X
1941(First,)X
2150(we)X
2277(put)X
2415(the)X
2548(previously)X
2945(developed)X
3332(theory)X
3582(in)X
3675(a)X
3738(continu-)X
576 1564(ous)N
729(setting.)X
1018(Let)X
1167(the)X
1307(sample)X
1589(space)X
2 f
1816(S)X
9 f
1881(=)X
2 f
1943({)X
1 f
1978(0,)X
2065(1)X
2 f
(}*)S
9 f
2202(\310)X
2 f
2270({)X
1 f
2305(0,)X
2392(1)X
2 f
(})S
1 f
8 s
9 f
2471 1529(\245)N
11 s
1 f
2524 1564(,)N
2578(the)X
2718(set)X
2848(of)X
2953(all)X
3074(\256nite)X
3288(and)X
3447(one-way)X
3781(in\256nite)X
576 1724(binary)N
827(sequences.)X
1253(Let)X
1397(a)X
1462(cylinder)X
9 f
1777(G)X
2 f
8 s
1830 1741(x)N
11 s
9 f
1879 1724(=)N
2 f
1941({xy)X
1 f
2068(:)X
2 f
2107(y)X
9 f
2167(\316)X
2 f
2244(S)X
2295(})X
1 f
2330(,)X
2378(the)X
2512(set)X
2636(of)X
2735(all)X
2850(elements)X
3189(from)X
2 f
3385(S)X
1 f
3454(that)X
3612(start)X
3789(with)X
2 f
3971(x)X
1 f
4010(.)X
576 1884(A)N
661(function)X
9 f
977(m)X
1 f
1050(from)X
1243(cylinders)X
1588(to)X
1679(the)X
1809(real)X
1963(interval)X
2255([0,)X
2371(1])X
2466(is)X
2547(called)X
2780(a)X
2 f
2841(semimeasure)X
1 f
3321(if)X
576 2072(\(a\))N
9 f
776(m)X
1 f
827(\()X
2 f
856(S)X
1 f
907(\))X
9 f
958(\243)X
1 f
1028(1;)X
1119(and)X
576 2260(\(b\))N
9 f
776(m)X
1 f
827(\()X
9 f
856(G)X
2 f
8 s
909 2277(x)N
1 f
11 s
944 2260(\))N
9 f
995(\263)X
1065(m)X
1 f
1116(\()X
9 f
1145(G)X
2 f
8 s
1198 2277(x)N
1 f
1236(0)X
11 s
1275 2260(\))N
2 f
9 f
1318(+)X
1 f
9 f
1380(m)X
1 f
1431(\()X
9 f
1460(G)X
2 f
8 s
1513 2277(x)N
1 f
1551(1)X
11 s
1590 2260(\))N
1641(.)X
776 2448(A)N
873(semimeasure)X
1367(is)X
1460(called)X
1705(a)X
2 f
1778(measure)X
1 f
2109(if)X
2197(equality)X
2516(holds)X
2741(in)X
2844(\(a\))X
2975(and)X
3136(\(b\).)X
3294(A)X
3392(semimeasure)X
9 f
3887(m)X
1 f
3973(is)X
2 f
576 2608(\(semi\)computable)N
1 f
1232(if)X
2 f
1319(f)X
1 f
1372(\()X
2 f
1401(x)X
1 f
1447(\))X
2 f
9 f
1490(=)X
1 f
9 f
1552(m)X
1 f
1603(\()X
9 f
1632(G)X
2 f
8 s
1685 2625(x)N
1 f
11 s
1720 2608(\))N
1775(is)X
1860(\(semi\)computable.)X
2567(Note)X
2764(that)X
2 f
2930(f)X
1 f
2981(needs)X
3206(to)X
3300(satisfy)X
3555(\(a\))X
3677(and)X
3829(\(b\).)X
3978(It)X
576 2768(is)N
660(more)X
866(convenient,)X
1300(and)X
1452(customary)X
1844(in)X
1939(this)X
2093(area,)X
2287(to)X
2382(simply)X
2649(write)X
9 f
2856(m)X
1 f
2907(\()X
2 f
2936(x)X
1 f
2982(\))X
3037(instead)X
3313(of)X
9 f
3412(m)X
1 f
3463(\()X
9 f
3492(G)X
2 f
8 s
3545 2785(x)N
1 f
11 s
3580 2768(\).)N
3679(The)X
3842(prob-)X
576 2928(lem)N
731(was)X
889(that)X
1044(the)X
1174(proper)X
1425(a)X
1486(priori)X
1704(probabilities)X
9 f
2168(m)X
1 f
2241(in)X
2332(formula)X
2633(\(2\))X
2757(are)X
2886(not)X
3021(known.)X
776 3116(We)N
923(modify)X
1203(the)X
1336(Turing)X
1601(machines)X
1959(in)X
2053(the)X
2186(standard)X
2509(enumeration)X
2975(so)X
3078(that)X
3236(they)X
3413(correspond)X
3829(to)X
3924(the)X
576 3276(semicomputable)N
1177(measures.)X
776 3464(A)N
2 f
868(monotonic)X
1 f
1269(machine)X
2 f
1597(M)X
1 f
1699(is)X
1787(a)X
1855(three)X
2060(tape)X
2236(machine)X
2564(similar)X
2840(to)X
2939(the)X
3077(pre\256x)X
3312(machine)X
3641(we)X
3773(de\256ned)X
576 3624(before,)N
850(but)X
990(now)X
1168(for)X
1297(all)X
1413(\256nite)X
1622(\(binary\))X
1932(inputs)X
2 f
2175(p)X
1 f
2246(and)X
2 f
2400(q)X
1 f
(,)S
2493(if)X
2 f
2574(p)X
1 f
2645(is)X
2731(a)X
2797(pre\256x)X
3029(of)X
2 f
3129(q)X
1 f
(,)S
3222(then)X
2 f
3401(M)X
1 f
3488(\()X
2 f
3517(q)X
1 f
3568(\))X
2 f
9 f
3611(=)X
2 f
3673(M)X
1 f
3760(\()X
2 f
3789(p)X
1 f
3840(\))X
2 f
3869(r)X
1 f
3930(for)X
576 3784(some)N
2 f
795(r)X
1 f
862(in)X
2 f
964(S)X
1 f
(.)S
1063(\(For)X
1247(convenience)X
1720(we)X
1855(de\256ne)X
2103(the)X
2 f
2244(M)X
1 f
2331(\()X
2 f
2360(p)X
1 f
2411(\))X
2473(as)X
2579(the)X
2720(contents)X
3047(of)X
3153(the)X
3294(output)X
3554(tape)X
3735(when)X
2 f
3959(M)X
1 f
576 3944(reads)N
786(the)X
919(next)X
1096(symbol)X
1381(after)X
2 f
1567(p)X
1 f
(.)S
1658(If)X
2 f
1741(M)X
1 f
1839(doesn't)X
2123(halt)X
2281(then)X
2 f
2458(M)X
1 f
2545(\()X
2 f
2574(p)X
1 f
2625(\))X
2679(can)X
2826(be)X
2934(\256nite)X
3141(or)X
3239(one-way)X
3566(in\256nite.\))X
3914(Let)X
2 f
576 4104(U)N
1 f
662(be)X
768(the)X
2 f
899(universal)X
1 f
1250(monotonic)X
1651(machine,)X
1995(in)X
2087(the)X
2218(same)X
2422(way)X
2592(as)X
2689(we)X
2815(have)X
3005(already)X
3288(met)X
3445(universal)X
3792(Turing)X
576 4264(machines)N
931(and)X
1080(universal)X
1425(pre\256x)X
1652(machines.)X
776 4452(The)N
2 f
935(universal)X
1285(semicomputable)X
1879(semimeasure)X
1 f
2359(is)X
2440(de\256ned)X
2721(as)X
3 f
1911 4692(M)N
1 f
1994(\()X
2 f
2023(x)X
1 f
2069(\))X
2 f
9 f
2120(=)X
1 f
2 f
8 s
2190 4782(U)N
1 f
2246(\()X
2 f
2267(p)X
1 f
2304(\))X
2 f
9 f
2335(\316)X
1 f
9 f
2391(G)X
2 f
5 s
2430 4794(x)N
1 f
16 s
9 f
2283 4718(S)N
11 s
1 f
2474 4692(2)N
2 f
8 s
9 f
4657(-)Y
2 f
2553(l)X
1 f
2581(\()X
2 f
2602(p)X
1 f
2639(\))X
11 s
2667 4692(,)N
576 5017(i.e.,)N
3 f
732(M)X
1 f
815(\()X
2 f
844(x)X
1 f
890(\))X
945(is)X
1030(the)X
2 f
1165(a)X
1236(priori)X
1469(probability)X
1 f
1889(that)X
2049(the)X
2184(output)X
2437(of)X
2537(the)X
2672(reference)X
3025(monotonic)X
3430(machine)X
2 f
3756(U)X
1 f
3846(starts)X
576 5177(with)N
2 f
762(x)X
1 f
801(.)X
852(Just)X
1018(as)X
1120(in)X
1218(the)X
1355(discrete)X
1658(case,)X
1860(one)X
2016(can)X
2167(show)X
2381(that)X
2543(for)X
2674(each)X
2864(semicomputable)X
3471(semimeasure)X
9 f
3959(m)X
1 f
4010(,)X
9 s
10 f
576 5265(hhhhhhhhhhhhhhhh)N
9 f
576 5353(d)N
6 s
1 f
612 5367(0)N
9 s
642 5353(\()N
2 f
(x)S
1 f
9 f
722(|)X
2 f
760(P)X
1 f
810(\))X
2 f
9 f
846(=)X
2 f
898(n)X
9 f
952(-)X
2 f
1004(K)X
1 f
1064(\()X
2 f
(x)S
1 f
9 f
1144(|)X
2 f
1182(n)X
1 f
1224(\))X
2 f
9 f
1260(+)X
2 f
1312(O)X
1 f
1376(\(1\),)X
1501(where)X
2 f
1700(n)X
9 f
1754(=)X
2 f
1806(l)X
1 f
1838(\()X
2 f
(x)S
1 f
1900(\).)X
1983(Such)X
2150(a)X
2205(\(universal)X
2516(Martin-L)X
2784 5310(.)N
2798(.)X
2780 5353(of\))N
2887(test)X
3010(is)X
3081(needed,)X
3326(since)X
3497(otherwise)X
3800(we)X
3908(can-)X
576 5441(not)N
703(distinguish,)X
1072(for)X
1191(instance,)X
1480(between)X
1755(randomness)X
2134(and)X
2273(nonrandomness)X
2760(of)X
2854(samples)X
3120(from)X
3294(the)X
3416(uniform)X
3682(distribution.)X
576 5529(\(Clearly,)N
848(the)X
954(word)X
1120(C)X
1187(o)X
1242(n)X
1297(s)X
1344(t)X
1383(a)X
1434(n)X
1489(t)X
1528(i)X
1567(n)X
1622(o)X
1677(p)X
1732(l)X
1771(e)X
1822(is)X
1889(not)X
2000(a)X
2051(random)X
2290(14-letter)X
2553(word.)X
2738(The)X
2869(probability)X
3204(of)X
3283(seeing)X
3486(it)X
3545(somewhere)X
3892(writ-)X
576 5617(ten)N
682(is)X
748(decidedly)X
1046(greater)X
1264(than)X
1406(128)X
2 f
6 s
9 f
5589(-)Y
1 f
1540(14)X
9 s
1594 5617(,)N
1630(say,)X
1762(for)X
1864(a)X
1914(randomly)X
2208(selected)X
2458(fourteen)X
2716(letter)X
2882(ASCII)X
3088(word.\))X
ep
%%Page: 22 22
bp
9 s 0 xH 0 xS 1 f
11 s
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(22)X
2370(-)X
576 736(there)N
774(exists)X
997(a)X
1058(constant)X
2 f
1374(c)X
1 f
1413(,)X
1457(such)X
1640(that)X
1795(for)X
1919(all)X
2 f
2030(x)X
9 f
2090(\316)X
2 f
2167({)X
1 f
2202(0,)X
2289(1)X
2 f
(})S
1 f
2368(,)X
2412(we)X
2536(have)X
3 f
2046 976(M)N
1 f
2129(\()X
2 f
2158(x)X
1 f
2204(\))X
9 f
2255(\263)X
2 f
2325(c)X
1 f
9 f
2385(m)X
1 f
2436(\()X
2 f
2465(x)X
1 f
2511(\))X
2 f
2540(.)X
1 f
576 1216(An)N
714(alternative)X
1118(approach)X
1471(to)X
1572(de\256ning)X
2 f
1893(a)X
1969(priori)X
2207(probability)X
1 f
2632(was)X
2800(taken)X
3023(by)X
3143(Cover)X
3390([)X
3451(Cover)X
3698(gambling)X
576 1376(schemes)N
896(who)X
1069(de\256ned)X
3 f
1739 1616(Mc)N
1 f
1861(\()X
2 f
1890(x)X
1 f
1936(\))X
2 f
9 f
1987(=)X
1 f
16 s
9 f
2057 1642(S)N
2 f
11 s
2133 1616({)N
3 f
2168(m)X
1 f
2241(\()X
2 f
2270(xy)X
1 f
2355(\):)X
2 f
2423(y)X
9 f
2483(\316)X
2 f
2560({)X
1 f
2595(0,)X
2682(1)X
2 f
(}*)S
2812(}.)X
1 f
576 1864(This)N
755(function)X
1071(has)X
1210(related)X
1472(properties)X
1846(to)X
3 f
1937(M)X
1 f
2020(.)X
3 f
776 2052(Solomonoff's)N
1297(Predictor.)X
1 f
1726(Instead)X
2008(of)X
2109(using)X
2328(formula)X
2635(\(2\),)X
2787(we)X
2917(estimate)X
3240(the)X
3376(conditional)X
3803(proba-)X
576 2212(bility)N
2 f
786(P)X
1 f
854(\()X
2 f
883(xy)X
1 f
9 f
989(|)X
2 f
1035(x)X
1 f
1081(\))X
1132(that)X
1287(the)X
1417(next)X
1591(segment)X
1907(after)X
2 f
2090(x)X
1 f
2151(is)X
2 f
2232(y)X
1 f
2293(by)X
2403(the)X
2533(expression)X
3 f
2200 2548(M)N
1 f
2283(\()X
2 f
2312(x)X
1 f
2358(\))X
3 f
2170 2435(M)N
1 f
2253(\()X
2 f
2282(x)X
2342(y)X
1 f
2388(\))X
10 f
2156 2461(h)N
2166(hhhhhh)X
2 f
2438 2487(.)N
1 f
3930(\(5\))X
576 2771(Now)N
769(let)X
9 f
881(m)X
1 f
955(in)X
1047(Formula)X
1369(\(2\))X
1495(be)X
1602(an)X
1709(arbitrary)X
2036(computable)X
2472(measure.)X
2833(This)X
3014(case)X
3189(includes)X
3507(all)X
3620(computable)X
576 2931(sequences,)N
976(as)X
1071(well)X
1245(as)X
1340(many)X
1558(Bernouilli)X
1939(sequences.)X
3 f
776 3119(Justi\256cation.)N
1 f
1308(Solomonoff)X
1758([)X
1816(Solomonoff)X
2266(IEEE)X
2486(1978)X
2691(showed)X
2988(that)X
3150(convergence)X
3623(of)X
3725(the)X
3862(error)X
576 3279(made)N
801(by)X
923(the)X
1065(estimator)X
1428(is)X
1521(very)X
1711(fast,)X
1894(in)X
1997(the)X
2139(following)X
2516(sense.)X
2784(If)X
9 f
2876(m)X
1 f
2961(is)X
3054(the)X
3196(actual)X
3440(prior)X
3644(probability)X
576 3439(\(measure\))N
952(over)X
1133(the)X
1266(sample)X
1541(space)X
2 f
1761({)X
1 f
1796(0,)X
1883(1)X
2 f
(})S
1 f
1962(,)X
2009(than)X
2186(we)X
2313(obviously)X
2686(cannot)X
2946(do)X
3060(better)X
3287(in)X
3382(predicting)X
3766(a)X
3831(`0')X
3959(or)X
576 3599(`1')N
700(after)X
883(an)X
988(initial)X
1218(segment)X
2 f
1534(x)X
1 f
1595(than)X
1769(using)X
1982(the)X
2112(inferred)X
2412(probability)X
9 f
2042 3935(m)N
1 f
2093(\()X
2 f
2122(x)X
1 f
2168(\))X
9 f
2020 3822(m)N
1 f
2071(\()X
2 f
2100(xa)X
1 f
2190(\))X
10 f
2006 3848(h)N
2012(hhhhh)X
1 f
2240 3874(,)N
2 f
2313(a)X
1 f
2 f
9 f
2386(=)X
1 f
2456(0,)X
2543(1)X
2 f
(.)S
1 f
576 4158(To)N
704(estimate)X
1029(how)X
1210(much)X
1436(worse)X
1675(it)X
1755(is)X
1844(to)X
1943(use)X
3 f
2090(M)X
1 f
2203(instead)X
2483(of)X
9 f
2586(m)X
1 f
2667(we)X
2799(consider)X
3128(the)X
3267(difference)X
3654(in)X
3754(inferred)X
576 4318(probabilities.)N
1090(Let)X
2 f
1236(S)X
8 s
4335(n)Y
1 f
11 s
1347 4318(denotes)N
1644(the)X
9 f
1780(m)X
1 f
1831(-expectation)X
2295(of)X
2396(the)X
2532(squared)X
2833(difference)X
3217(between)X
3538(the)X
9 f
3674(m)X
1 f
3725(-inferred)X
576 4478(probability)N
986(and)X
1135(the)X
3 f
1265(M)X
1 f
1348(-inferred)X
1677(probability,)X
2109(of)X
2204(`0')X
2328(occurring)X
2687(as)X
2782(the)X
2 f
2912(n)X
9 f
2977(+)X
1 f
3039(1th)X
3174(symbol:)X
2 f
1659 4779(S)N
8 s
4796(n)Y
1 f
11 s
2 f
9 f
1764 4779(=)N
1 f
2 f
8 s
1834 4869(l)N
1 f
1862(\()X
2 f
1883(x)X
1 f
1916(\))X
2 f
9 f
1947(=)X
2 f
1992(n)X
1 f
16 s
9 f
1891 4805(S)N
11 s
2038 4779(m)N
1 f
2089(\()X
2 f
2118(x)X
1 f
2164(\))X
10 f
2207 4691(I)N
2207 4779(J)N
2207 4867(L)N
3 f
2280 4840(M)N
1 f
2363(\()X
2 f
2392(x)X
1 f
2438(\))X
3 f
2251 4727(M)N
1 f
2334(\()X
2 f
2363(x)X
1 f
2423(0\))X
10 f
2237 4753(h)N
2245(hhhhhh)X
2 f
9 f
2539 4779(-)N
1 f
9 f
2660 4840(m)N
1 f
2711(\()X
2 f
2740(x)X
1 f
2786(\))X
9 f
2631 4727(m)N
1 f
2682(\()X
2 f
2711(x)X
1 f
2771(0\))X
10 f
2617 4753(h)N
2637(hhhhh)X
2865 4691(M)N
2865 4779(J)N
2865 4867(O)N
8 s
1 f
4664(2)Y
2 f
11 s
2926 4779(.)N
1 f
576 5098(Then)N
16 s
9 f
788 5124(S)N
2 f
8 s
864 5142(n)N
11 s
917 5098(S)N
8 s
5115(n)Y
1 f
11 s
9 f
1014 5098(\243)N
2 f
1076(K)X
1 f
1149(\()X
9 f
1178(m)X
1 f
1229(\))X
2 f
1265(/)X
1 f
1297(2.)X
1417(Here,)X
2 f
1641(K)X
1 f
1714(\()X
9 f
1743(m)X
1 f
1794(\))X
1855(is)X
1946(the)X
2086(Kolmogorov)X
2568(complexity)X
2998(of)X
3103(the)X
3243(index)X
2 f
3471(i)X
1 f
3496(,)X
3550(where)X
2 f
3796(T)X
8 s
3845 5115(i)N
1 f
11 s
3902 5098(is)N
3993(a)X
576 5284(Turing)N
840(machine)X
1163(computing)X
9 f
1565(m)X
1 f
1616(.)X
1684(Therefore,)X
2 f
2076(S)X
8 s
5301(n)Y
1 f
11 s
2183 5284(converges)N
2563(to)X
2656(zero)X
2831(faster)X
3050(than)X
3226(1)X
2 f
3277(/)X
3309(n)X
1 f
(.)S
3421(In)X
3518(other)X
3723(words,)X
3982(it)X
576 5444(has)N
722(been)X
917(rigorously)X
1308(proved)X
1581(that)X
1743(for)X
1874(the)X
2011(above)X
2250(estimator)X
2608(the)X
2 f
2746(expected)X
3084(error)X
1 f
3299(at)X
3393(the)X
2 f
3531(n)X
1 f
(th)S
3674(prediction)X
576 5604(converges)N
954(to)X
1045(zero)X
1218(faster)X
1435(than)X
1609(1)X
2 f
1660(/)X
1692(n)X
1 f
(!)S
776 5792(This)N
959(was)X
1122(improved)X
1487(by)X
1602(G)X
1680(\302)X
1665(acs)X
1804([)X
1860(%A)X
2023(P.)X
2121(G)X
2199(\302)X
2184(acs)X
2323(%T)X
2477(Personal)X
2807(Communication)X
3404(as)X
3504(follows.)X
3839(If)X
3924(the)X
ep
%%Page: 23 23
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(23)X
2370(-)X
576 736(length)N
819(of)X
2 f
914(y)X
1 f
975(is)X
1056(\256xed,)X
1276(and)X
1425(the)X
1555(length)X
1798(of)X
2 f
1893(x)X
1 f
1954(grows)X
2190(to)X
2281(in\256nity,)X
2581(then)X
9 f
1998 1072(m)N
1 f
2049(\()X
2 f
2078(x)X
2138(y)X
1 f
2184(\))X
2 f
2220(/)X
1 f
9 f
2252(m)X
1 f
2303(\()X
2 f
2332(x)X
1 f
2378(\))X
3 f
1966 959(M)N
1 f
2049(\()X
2 f
2078(x)X
2138(y)X
1 f
2184(\))X
2 f
2220(/)X
3 f
2252(M)X
1 f
2335(\()X
2 f
2364(x)X
1 f
2410(\))X
10 f
1952 985(h)N
1968(hhhhhhhhhhh)X
9 f
2482 1011 MXY
(->)174 987 oc
1 f
2591(1,)X
576 1295(with)N
9 f
773(m)X
1 f
824(-probability)X
1281(one.)X
1470(In)X
1583(other)X
1804(words,)X
2080(the)X
2228(conditional)X
2666(a)X
2745(priori)X
2982(probability)X
3411(is)X
3511(almost)X
3788(always)X
576 1455(asymptotically)N
1119(equal)X
1332(to)X
1423(the)X
1553(conditional)X
1973(probability.)X
776 1643(With)N
975(respect)X
1246(to)X
1337(the)X
1467(discrete)X
1763(sample)X
2035(space)X
2252(approach)X
2596(taken)X
2809(before,)X
3077(one)X
3226(can)X
3370(show)X
3577(that:)X
2 f
9 f
1459 1883(-)N
1 f
1521(log)X
3 f
1648(M)X
1 f
1731(\()X
2 f
1760(x)X
1 f
1806(\))X
2 f
9 f
1857(=)X
1 f
2 f
9 f
1927(-)X
1 f
1989(log)X
3 f
2116(Mc)X
1 f
2238(\()X
2 f
2267(x)X
1 f
2313(\))X
2 f
9 f
2364(=)X
1 f
2 f
2434(K)X
1 f
2507(\()X
2 f
2536(x)X
1 f
2582(\))X
2 f
9 f
2625(+)X
2 f
2687(O)X
1 f
2764(\(log)X
2 f
2920(K)X
1 f
2993(\()X
2 f
3022(x)X
1 f
3068(\)\).)X
3930(\(6\))X
3 f
576 2283(2.4.2.)N
818(Conclusions)X
1 f
576 2471(On)N
706(the)X
837(positive)X
1141(side)X
1307(we)X
1433(have)X
1623(achieved)X
1960(the)X
2092(following.)X
2503(Bayes')X
2771(rule)X
2932(using)X
3147(the)X
3279(universal)X
3626(prior)X
3821(distri-)X
576 2631(bution)N
830(gives)X
1044(an)X
1155(objective)X
1507(interpretation)X
2010(to)X
2106(Occam's)X
2445(razor)X
2652(principle.)X
3015(Namely,)X
3343(if)X
3424(several)X
3700(programs)X
576 2791(could)N
797(generate)X
2 f
1120(S)X
1 f
1178(0)X
1247(then)X
1424(the)X
1557(shortest)X
1856(one)X
2008(is)X
2092(used)X
2278(\(for)X
2434(the)X
2567(prior)X
2763(probability\),)X
3227(and)X
3379(further)X
3643(if)X
2 f
3722(S)X
1 f
3780(0)X
3850(has)X
3993(a)X
576 2951(shorter)N
847(program)X
1172(than)X
2 f
1351(S)X
1 f
1409(1)X
1480(then)X
2 f
1659(S)X
1 f
1717(0)X
1788(is)X
1874(preferred)X
2222(\()X
2 f
2251(i.e.)X
1 f
2386(predict)X
2658(0)X
2729(with)X
2913(higher)X
3165(probability)X
3580(than)X
3758(predict-)X
576 3111(ing)N
711(1)X
777(after)X
960(seeing)X
2 f
1207(S)X
1 f
(\).)S
1324(Bayes')X
1590(rule)X
1750(via)X
1881(the)X
2012(universal)X
2358(prior)X
2552(distribution)X
2983(also)X
3148(satis\256es)X
3450(Epicurus')X
3815(multi-)X
576 3271(ple)N
721(explanations)X
1203(dictum,)X
1507(since)X
1724(we)X
1862(do)X
1986(not)X
2135(select)X
2372(a)X
2447(single)X
2694(hypothesis)X
3107(after)X
3304(considering)X
3751(the)X
3895(evi-)X
576 3431(dence,)N
826(but)X
962(maintain)X
1295(all)X
1407(hypotheses)X
1821(consistent)X
2197(with)X
2377(the)X
2508(evidence)X
2844(and)X
2994(just)X
3146(transform)X
3512(the)X
3644(probability)X
576 3591(distribution)N
1018(on)X
1140(the)X
1282(hypotheses)X
1707(according)X
2088(to)X
2191(the)X
2333(evidence.)X
2724(Finally,)X
3031(there)X
3240(is)X
3332(mathematical)X
3842(proof)X
576 3751(that)N
741(Solomonoff's)X
1257(inference)X
1616(procedure)X
1999(using)X
2222(the)X
2362(universal)X
2717(prior)X
2920(probability)X
3340(performs)X
3690(almost)X
3959(as)X
576 3911(good)N
774(as)X
869(the)X
999(one)X
1148(using)X
1361(the)X
1491(actual)X
1724(\(computable\))X
2216(prior)X
2409(probability.)X
776 4099(On)N
918(the)X
1061(negative)X
1395(side)X
1572(we)X
1709(know)X
1939(that)X
2107(Solomonoff's)X
2626(inference)X
2988(is)X
3082(not)X
3230(practicable)X
3652(in)X
3756(its)X
3876(pure)X
576 4259(form.)N
815(The)X
976(universal)X
1323(prior)X
1518(distributions)X
3 f
1984(m)X
1 f
2057(\()X
2 f
2086(x)X
1 f
2132(\))X
2185(for)X
2311(discrete)X
2609(sample)X
2883(spaces,)X
3158(and)X
3 f
3309(M)X
1 f
3392(\()X
2 f
3421(x)X
1 f
3467(\))X
3520(for)X
3645(continuous)X
576 4419(sample)N
854(spaces,)X
1133(are)X
1268(not)X
1409(computable,)X
1871(essentially)X
2272(because)X
2578(the)X
2715(Kolmogorov)X
3194(complexity)X
3621(is)X
3709(not)X
3851(com-)X
576 4579(putable.)N
911(However,)X
1285(we)X
1418(can)X
1571(compute)X
1906(approximations)X
2486(to)X
2 f
2586(K)X
1 f
2659(\()X
2 f
2688(x)X
1 f
2734(\),)X
3 f
2816(m)X
1 f
2889(\()X
2 f
2918(x)X
1 f
2964(\),)X
3046(and)X
3 f
3204(M)X
1 f
3287(\()X
2 f
3316(x)X
1 f
3362(\).)X
3466(It)X
3550(turns)X
3756(out)X
3899(that)X
576 4739(using)N
790(Solomonoff's)X
1298(inference)X
1649(principles)X
2021(with)X
2202(such)X
2387(computable)X
2823(approximations)X
3396(yields)X
3631(many)X
3851(other)X
576 4899(known)N
845(inference)X
1202(models)X
1487(or)X
1590(principles.)X
2012(In)X
2115(the)X
2253(next)X
2435(few)X
2596(sections,)X
2932(we)X
3064(derive)X
3314(or)X
3416(establish)X
3754(connec-)X
576 5059(tions)N
782(with)X
973(various)X
1266(well-known)X
1721(machine)X
2055(learning)X
2379(models)X
2669(and)X
2831(inductive)X
3195(inference)X
3557(paradigms)X
3959(or)X
576 5219(principles.)N
969(Thus)X
1168(we)X
1293(provide)X
1585(an)X
1691(alternative)X
2087(view)X
2281(of)X
2377(these)X
2581(models)X
2859(and)X
3009(principles)X
3379(from)X
3572(the)X
3702(lofty)X
3891(per-)X
576 5379(spective)N
887(of)X
982(Kolmogorov)X
1454(complexity.)X
ep
%%Page: 24 24
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(24)X
2370(-)X
3 f
576 736(3.)N
686(Gold's)X
958(Inductive)X
1337(Inference)X
1715(Paradigm)X
1 f
776 924(There)N
1007(are)X
1140(many)X
1362(different)X
1691(ways)X
1897(of)X
1996(formulating)X
2439(concrete)X
2763(inductive)X
3119(inference)X
3473(problems)X
3828(in)X
3924(the)X
576 1084(real)N
730(world.)X
1001(We)X
1145(will)X
1305(try)X
1425(to)X
1516(simplify)X
1833(matters)X
2115(as)X
2210(much)X
2428(as)X
2523(possible)X
2834(short)X
3032(of)X
3127(losing)X
3365(signi\256cance.)X
776 1272(\(i\))N
3 f
887(The)X
1062(class)X
1266(of)X
1367(rules)X
1 f
1581(we)X
1711(consider)X
2037(can)X
2187(be)X
2298(various)X
2585(classes)X
2857(of)X
2958(languages)X
3338(or)X
3439(functions,)X
3818(where)X
576 1432(we)N
707(restrict)X
981(ourselves)X
1342(to)X
1440(classes)X
1713(of)X
1815(recursive)X
2166(sets,)X
2349(context-free)X
2803(languages,)X
3206(regular)X
3484(sets)X
3644(and)X
3799(sets)X
3959(of)X
576 1592(\256nite)N
790(automata,)X
1168(and)X
1327(sets)X
1491(of)X
1596(Boolean)X
1922(formulae.)X
2316(We)X
2470(treat)X
2659(a)X
2730(language)X
2 f
3080(L)X
1 f
3161(as)X
3266(a)X
2 f
3337(function)X
3666(f)X
1 f
3724(using)X
3948(its)X
576 1752(characteristic)N
1068(function,)X
2 f
1406(i.e.)X
1 f
1514(,)X
2 f
1565(f)X
1 f
1618(\()X
2 f
1647(x)X
1 f
1693(\))X
2 f
9 f
1736(=)X
1 f
9 f
1798(c)X
2 f
8 s
1846 1769(L)N
1 f
11 s
1889 1752(\()N
2 f
1918(x)X
1 f
1964(\))X
2 f
9 f
1993(=)X
1 f
2041(1)X
2107(if)X
2 f
2183(x)X
9 f
2236(\316)X
2 f
2299(L)X
1 f
2348(,)X
2392(and)X
2541(0)X
2607(otherwise.)X
776 1940(\(ii\))N
3 f
908(The)X
1079(hypothesis)X
1499(space)X
1 f
1728(or)X
3 f
1825(rule)X
2001(space)X
1 f
2230(denoted)X
2533(by)X
3 f
2645(R)X
1 f
2733(speci\256es)X
3061(syntactically)X
3533(how)X
3709(each)X
3895(rule)X
576 2100(in)N
675(\(i\))X
788(should)X
1053(be)X
1165(represented.)X
1621(We)X
1772(\256x)X
1894(a)X
1962(standard)X
2289(enumeration)X
2759(of)X
2861(the)X
2998(representations)X
3560(for)X
3691(a)X
3759(class)X
3959(of)X
576 2260(rules,)N
3 f
794(R)X
2 f
9 f
871(=)X
2 f
933({R)X
1 f
8 s
1032 2277(1)N
11 s
1071 2260(,)N
2 f
1114(R)X
1 f
8 s
1178 2277(2)N
11 s
1217 2260(,)N
2 f
1246(...})X
1 f
1347(,)X
1394(and)X
1546(assume)X
1830(that)X
1988(each)X
2174(rule)X
2 f
2343(f)X
1 f
2393(has)X
2535(at)X
2625(least)X
2813(one)X
2966(description)X
3384(in)X
3479(the)X
3613(correspond-)X
576 2420(ing)N
721(hypothesis)X
1129(space.)X
1377(For)X
1530(example,)X
1882(the)X
2021(hypothesis)X
2429(space)X
2655(can)X
2808(be)X
2922(standard)X
3251(encodings)X
3639(of)X
3743(context-)X
576 2580(free)N
748(grammars,)X
1158(or)X
1267(standard)X
1601(encodings)X
1994(of)X
2103(Finite)X
2346(Automata.)X
2752(In)X
2861(any)X
3024(case,)X
3233(it)X
3319(is)X
3414(assumed)X
3754(that)X
3924(the)X
576 2740(hypothesis)N
981(space)X
1204(is)X
1291(effectively)X
1696(enumerable)X
2134(\(so)X
2268(it)X
2345(cannot)X
2607(be)X
2717(the)X
2852(set)X
2977(of)X
3077(all)X
3193(halting)X
3466(Turing)X
3733(machine)X
576 2900(codes\).)N
879(For)X
1031(convenience,)X
1523(this)X
1681(enumeration)X
2152(of)X
2255(hypotheses)X
2 f
2676(R)X
1 f
8 s
2740 2917(1)N
11 s
2779 2900(,)N
2 f
2822(R)X
1 f
8 s
2886 2917(2)N
11 s
2925 2900(,)N
2 f
2954(...)X
1 f
3073(consists)X
3383(of)X
3487(codes)X
3718(for)X
3851(algo-)X
576 3060(rithms)N
829(to)X
925(compute)X
1256(recursive)X
1605(functions)X
2 f
1967(f)X
1 f
8 s
2002 3077(1)N
11 s
2041 3060(,)N
2 f
2091(f)X
1 f
8 s
2126 3077(2)N
11 s
2165 3060(,)N
2 f
2194(...)X
1 f
2287(\(languages)X
2695(are)X
2829(represented)X
3260(by)X
3374(their)X
3562(characteristic)X
576 3220(functions\).)N
776 3408(\(iii\))N
3 f
941(The)X
1120(presentation)X
1621(of)X
1726(examples)X
1 f
2105(is)X
2196(vital)X
2386(to)X
2487(the)X
2627(inference)X
2986(process.)X
3303(We)X
3457(choose)X
3734(the)X
3875(sim-)X
576 3568(plest,)N
794(and)X
950(yet)X
1087(most)X
1288(general,)X
1598(form)X
1798(of)X
1900(data)X
2076(presentation.)X
2558(For)X
2709(a)X
2777(function)X
2 f
3107(f)X
1 f
3161(to)X
3259(be)X
3371(inferred,)X
3700(there)X
3905(is)X
3993(a)X
576 3728(\256xed)N
778(in\256nite)X
1055(sequence)X
1403(of)X
2 f
1502(examples)X
1 f
1855(\()X
2 f
1884(s)X
1 f
8 s
1928 3745(1)N
11 s
1967 3728(,)N
2 f
2017(f)X
1 f
2070(\()X
2 f
2099(s)X
1 f
8 s
2143 3745(1)N
11 s
2182 3728(\)\),)N
2283(\()X
2 f
2312(s)X
1 f
8 s
2356 3745(2)N
11 s
2395 3728(,)N
2 f
2445(f)X
1 f
2498(\()X
2 f
2527(s)X
1 f
8 s
2571 3745(2)N
11 s
2610 3728(\)\),)N
2697(....)X
2834(When)X
2 f
3078(f)X
9 f
3131(=)X
1 f
9 f
3179(c)X
2 f
8 s
3227 3745(L)N
1 f
11 s
3270 3728(,)N
3319(we)X
3448(have)X
9 f
3641(c)X
2 f
8 s
3689 3745(L)N
1 f
11 s
3732 3728(\()N
2 f
3761(s)X
1 f
3802(\))X
2 f
9 f
3845(=)X
1 f
3907(1)X
3978(if)X
2 f
576 3888(s)N
9 f
631(\316)X
2 f
708(L)X
1 f
779(\()X
2 f
808(s)X
1 f
864(is)X
945(a)X
1006(positive)X
1308(example)X
1629(of)X
2 f
1724(L)X
1 f
1773(\))X
1824(and)X
9 f
1973(c)X
2 f
8 s
2021 3905(L)N
1 f
11 s
2064 3888(\()N
2 f
2093(s)X
1 f
2134(\))X
2 f
9 f
2177(=)X
1 f
2239(0)X
2305(otherwise)X
2669(\()X
2 f
2698(s)X
1 f
2754(is)X
2835(a)X
2896(negative)X
3217(example)X
3538(of)X
2 f
3633(L)X
1 f
3682(\).)X
776 4076(A)N
861(rule)X
1020(\(or)X
1144(function\))X
2 f
1496(f)X
1 f
1543(is)X
1624(said)X
1788(to)X
1879(be)X
2 f
1984(consistent)X
1 f
2359(with)X
2538(the)X
2668(initial)X
2898(segment)X
3214(of)X
3309(examples)X
2 f
1831 4316(S)N
1 f
2 f
9 f
1904(=)X
1 f
1974(\()X
2 f
2003(s)X
1 f
8 s
2047 4333(1)N
11 s
2086 4316(,)N
2 f
2129(a)X
1 f
8 s
2183 4333(1)N
11 s
2222 4316(\))N
2265(,)X
2309(.)X
2353(.)X
2397(.)X
2441(,)X
2491(\()X
2 f
2520(s)X
8 s
2554 4333(n)N
1 f
11 s
2593 4316(,)N
2 f
2636(a)X
8 s
4333(n)Y
1 f
11 s
2719 4316(\),)N
3930(\(7\))X
576 4556(if)N
2 f
668(f)X
1 f
721(\()X
2 f
750(s)X
8 s
784 4573(i)N
1 f
11 s
809 4556(\))N
2 f
9 f
852(=)X
2 f
914(a)X
8 s
4573(i)Y
1 f
11 s
983 4556(,)N
2 f
1036(i)X
9 f
1082(=)X
1 f
1130(1,)X
1203(..,)X
2 f
1276(n)X
1 f
(.)S
1373(We)X
1526(require)X
1806(that)X
2 f
1970(all)X
1 f
2095(strings)X
2361(will)X
2530(eventually)X
2929(appear)X
3194(as)X
3299(\256rst)X
3468(component)X
3892(in)X
3993(a)X
576 4716(pair)N
735(in)X
2 f
826(S)X
1 f
(.)S
914(The)X
1073(last)X
1218(assumption)X
1642(is)X
1745(strong,)X
2009(but)X
2144(is)X
2225(essential)X
2551(to)X
2642(the)X
2772(Gold)X
2970(paradigm.)X
3 f
776 4904(How)N
974(to)X
1069(infer)X
1273(a)X
1340(rule.)X
1 f
1559(By)X
1685(\(ii\),)X
1838(there)X
2037(is)X
2119(an)X
2225(effective)X
2556(enumeration)X
2 f
3027(f)X
1 f
8 s
3062 4921(1)N
11 s
3101 4904(,)N
2 f
3151(f)X
1 f
8 s
3186 4921(2)N
11 s
3225 4904(,)N
2 f
3254(...)X
1 f
3365(of)X
3461(partial)X
3710(recursive)X
576 5064(functions)N
946(corresponding)X
1491(to)X
1602(the)X
1752(enumeration)X
2235(of)X
2350(hypotheses.)X
2827(The)X
3005(a)X
3085(priori)X
3322(probability)X
3751(of)X
2 f
3872(f)X
8 s
3897 5081(k)N
1 f
11 s
3973 5064(is)N
3 f
576 5224(m)N
1 f
649(\()X
2 f
685(f)X
8 s
710 5241(k)N
1 f
11 s
745 5224(\))N
2 f
9 f
788(=)X
3 f
850(m)X
1 f
923(\()X
2 f
952(k)X
1 f
998(\).)X
1093(\(Actually,)X
3 f
1470(m)X
1 f
1543(\()X
2 f
1579(f)X
8 s
1604 5241(k)N
1 f
11 s
1639 5224(\))N
2 f
9 f
1682(=)X
2 f
1744(c)X
3 f
1804(m)X
1 f
1877(\()X
2 f
1906(k)X
1 f
1952(\),)X
2025(for)X
2149(some)X
2358(constant)X
2 f
2675(c)X
1 f
2737(depending)X
3127(on)X
3238(the)X
3369(effective)X
3700(enumera-)X
576 5384(tion)N
738(involved,)X
1093(but)X
1230(not)X
1367(depending)X
1758(on)X
2 f
1869(n)X
1 f
(.)S
1980(To)X
2101(assume)X
2383(that)X
2 f
2539(c)X
9 f
2599(=)X
1 f
2661(1)X
2728(makes)X
2976(no)X
3087(difference)X
3466(in)X
3558(the)X
3689(following)X
576 5544(discussion.\))N
1041(We)X
1188(are)X
1320(given)X
1541(an)X
1649(in\256nite)X
1925(sequence)X
2272(of)X
2370(examples)X
2728(representing)X
3189(the)X
3323(rule)X
3486(or)X
3585(function)X
2 f
3912(f)X
1 f
3963(to)X
576 5704(be)N
691(learned.)X
1026(According)X
1429(to)X
1530(Bayes')X
1806(rule)X
1974(\(1\),)X
2129(for)X
2 f
2262(k)X
9 f
2322(=)X
1 f
2384(1,)X
2471(2,)X
2544(...,)X
2663(the)X
2802(inferred)X
3111(probability)X
3530(of)X
2 f
3641(f)X
8 s
3666 5721(k)N
1 f
11 s
3732 5704(after)N
3924(the)X
576 5864(sequence)N
920(of)X
1015(examples)X
1370(\(7\))X
1494(is)X
1575(given)X
1793(by:)X
ep
%%Page: 25 25
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(25)X
2370(-)X
2 f
1109 788(P)N
1 f
1177(\()X
2 f
1213(f)X
8 s
1238 805(k)N
11 s
9 f
1287 788(=)N
2 f
1356(f)X
1 f
9 f
1402(|)X
2 f
1455(f)X
1 f
1487(\()X
2 f
1516(s)X
8 s
1550 805(i)N
1 f
11 s
1575 788(\))N
2 f
9 f
1618(=)X
2 f
1680(a)X
8 s
805(i)Y
1 f
11 s
1749 788(,)N
2 f
1792(i)X
9 f
1838(=)X
1 f
1900(1,)X
1973(..,)X
2 f
2046(n)X
1 f
2097(\))X
2 f
9 f
2148(=)X
1 f
16 s
9 f
2327 875(S)N
2 f
11 s
2417 849({)N
3 f
2452(m)X
1 f
2525(\()X
2 f
2561(j)X
1 f
2593(\):)X
2 f
2668(f)X
8 s
2698 866(j)N
1 f
11 s
2723 849(\()N
2 f
2752(s)X
8 s
2786 866(i)N
1 f
11 s
2811 849(\))N
2 f
9 f
2854(=)X
2 f
2916(a)X
8 s
866(i)Y
1 f
11 s
2985 849(,)N
2 f
3028(i)X
9 f
3074(=)X
1 f
3136(1,)X
3209(..,)X
2 f
3282(n)X
3333(})X
2240 718(P)N
1 f
2308(\()X
2 f
2344(f)X
1 f
2376(\()X
2 f
2405(s)X
8 s
2439 735(i)N
1 f
11 s
2464 718(\))N
2 f
9 f
2507(=)X
2 f
2569(a)X
8 s
735(i)Y
1 f
11 s
2638 718(,)N
2 f
2681(i)X
9 f
2727(=)X
1 f
2789(1,)X
2862(..,)X
2 f
2935(n)X
1 f
9 f
3007(|)X
2 f
3060(f)X
8 s
3085 735(k)N
11 s
9 f
3134 718(=)N
2 f
3203(f)X
1 f
3235(\))X
3 f
3278(m)X
1 f
3351(\()X
2 f
3380(k)X
1 f
3426(\))X
10 f
2226 762(h)N
2236(hhhhhhhhhhhhhhhhhhhhhhhhhhhh)X
2 f
3476 788(.)N
1 f
3930(\(8\))X
576 1098(Cf.)N
708(also)X
872([)X
924(Cover)X
1162(1974)X
1361(gambling)X
1718(])X
1770([)X
1822(Cover)X
2060(Impact)X
2328(In)X
2424(the)X
2555(numerator)X
2940(of)X
3036(the)X
3167(right-hand)X
3557(term,)X
3764(the)X
3895(\256rst)X
576 1258(factor)N
808(is)X
894(zero)X
1072(or)X
1172(one)X
1326(depending)X
1720(on)X
1835(whether)X
2 f
2152(f)X
8 s
2177 1275(k)N
1 f
11 s
2239 1258(is)N
2325(consistent)X
2705(with)X
2 f
2889(S)X
1 f
(,)S
2982(and)X
3136(the)X
3271(second)X
3542(factor)X
3774(is)X
3859(the)X
3993(a)X
576 1418(priori)N
795(probability)X
1206(of)X
2 f
1309(f)X
8 s
1334 1435(k)N
1 f
11 s
1369 1418(.)N
1436(The)X
1596(denominator)X
2066(is)X
2149(a)X
2212(normalizing)X
2663(term)X
2849(giving)X
3099(the)X
3231(combined)X
3603(a)X
3666(priori)X
3886(pro-)X
576 1578(bability)N
873(of)X
972(all)X
1087(rules)X
1284(consistent)X
1663(with)X
2 f
1846(S)X
1 f
(.)S
1960(With)X
2163(increasing)X
2 f
2551(n)X
1 f
(,)S
2643(the)X
2777(denominator)X
3249(term)X
3437(is)X
3521(monotonically)X
576 1738(nonincreasing.)N
1120(Since)X
1345(all)X
1463(examples)X
1825(eventually)X
2222(appear)X
2485(in)X
2 f
2583(S)X
1 f
(,)S
2678(the)X
2815(denominator)X
3290(converges)X
3675(to)X
3773(a)X
3841(limit,)X
576 1898(say)N
2 f
720(d)X
1 f
9 f
785(\243)X
1 f
847(1.)X
962(For)X
1111(each)X
2 f
1299(k)X
1 f
1338(,)X
1386(the)X
1520(inferred)X
1824(probability)X
2 f
2245(f)X
8 s
2270 1915(k)N
1 f
11 s
2331 1898(is)N
2416(monotonically)X
2953(nondecreasing)X
3487(with)X
3670(increasing)X
2 f
576 2058(n)N
1 f
(,)S
669(until)X
2 f
866(f)X
8 s
891 2075(k)N
1 f
11 s
953 2058(is)N
1039(inconsistent)X
1488(with)X
1672(a)X
1738(new)X
1911(example,)X
2260(in)X
2357(which)X
2600(case)X
2779(it)X
2857(falls)X
3037(to)X
3134(zero)X
3313(and)X
3468(stays)X
3672(there)X
3876(hen-)X
576 2218(ceforth.)N
901(In)X
1006(the)X
1146(limit,)X
1369(only)X
1558(the)X
2 f
1705(f)X
8 s
1730 2235(k)N
1 f
11 s
1765 2218('s)N
1860(that)X
2025(are)X
2163(consistent)X
2547(with)X
2735(the)X
2874(sequence)X
3227(of)X
3331(presented)X
3699(examples)X
576 2378(have)N
780(positive)X
1098(inferred)X
1414(probability)X
3 f
1840(m)X
1 f
1913(\()X
2 f
1942(k)X
1 f
1988(\))X
2 f
2024(/)X
2056(d)X
1 f
(.)S
2160(By)X
2301(Theorem)X
2657(3,)X
2761(since)X
3 f
2980(m)X
1 f
3053(\()X
2 f
3082(k)X
1 f
3128(\))X
2 f
9 f
3171(=)X
1 f
9 f
3233(Q)X
1 f
3298(\(2)X
2 f
8 s
9 f
3371 2343(-)N
2 f
3406(K)X
1 f
3459(\()X
2 f
3480(k)X
1 f
3513(\))X
11 s
3541 2378(\),)N
3630(the)X
3777(highest)X
576 2538(inferred)N
898(probability)X
1330(is)X
1433(carried)X
1721(by)X
1853(the)X
2005(rule)X
2 f
2193(f)X
8 s
2218 2555(k)N
1 f
11 s
2297 2538(with)N
2498(least)X
2704(Kolmogorov)X
3198(complexity)X
3640(among)X
3924(the)X
576 2698(remaining)N
956(ones.)X
1161(Similar)X
1444(statements)X
1839(hold)X
2018(after)X
2201(each)X
2384(initial)X
2614(segment)X
2930(of)X
2 f
3025(n)X
1 f
3091(examples,)X
2 f
3468(n)X
9 f
3533(=)X
1 f
3595(1,)X
3668(2,)X
3741(....)X
776 2886(Reasoning)N
1187(inductively,)X
1646(we)X
1787(transform)X
2168(the)X
2 f
2315(a)X
2399(priori)X
1 f
2645(probability)X
3073(according)X
3460(to)X
3569(Formula)X
3908(\(8\),)X
576 3046(inferring)N
906(a)X
967(new)X
2 f
1135(posterior)X
1 f
1480(probability)X
1890(by)X
2000(the)X
2130(evidence)X
2465(of)X
2560(each)X
2743(initial)X
2973(segment)X
3289(of)X
3384(examples.)X
3761(At)X
3871(each)X
576 3206(step,)N
762(we)X
886(can)X
1031(select)X
1255(the)X
1386(rule)X
1546(with)X
1726(the)X
1857(highest)X
2135(inferred)X
2436(probability,)X
2869(and)X
3019(in)X
3111(the)X
3242(limit)X
3434(we)X
3559(have)X
3748(selected)X
576 3366(the)N
710(proper)X
965(rule.)X
1172(At)X
1286(each)X
1473(step)X
1641(we)X
1769(predict)X
2040(the)X
2173(rule)X
2335(with)X
2517(the)X
2650(highest)X
2930(inferred)X
3233(probability.)X
3690(Reformu-)X
576 3526(lating,)N
822(if)X
898(we)X
1022(want)X
1215(to)X
1306(infer)X
1494(a)X
1555(language)X
2 f
1895(L)X
1 f
1966(using)X
2179(this)X
2329(procedure,)X
2724(then:)X
776 3714(\(a\))N
909(The)X
1082(Bayesian)X
2 f
1441(a)X
1521(posteriori)X
1 f
1905(probability)X
2329(for)X
2467(the)X
2611(correct)X
2891(answer)X
3175(converges)X
3568(to)X
2 f
3674(c)X
1 f
3734(2)X
2 f
8 s
9 f
3679(-)Y
2 f
3813(l)X
1 f
3841(\()X
2 f
3862(p)X
1 f
3899(\))X
2 f
11 s
3934 3714(/)N
3966(d)X
1 f
(,)S
576 3874(where)N
2 f
819(p)X
1 f
892(is)X
980(the)X
1117(shortest)X
1419(program)X
1745(which)X
1988(the)X
2124(reference)X
2478(machine)X
2805(uses)X
2984(to)X
3081(simulate)X
2 f
3409(M)X
1 f
8 s
3492 3891(0)N
11 s
3531 3874(,)N
3581(where)X
2 f
3823(M)X
1 f
8 s
3906 3891(0)N
11 s
3973 3874(is)N
576 4034(the)N
707(smallest)X
1020(TM)X
1175(that)X
1331(accepts)X
2 f
1613(L)X
1 f
1662(.)X
1729(This)X
1909(correct)X
2176(answer)X
2447(will)X
2608(have)X
2797(the)X
2928(highest)X
3206(probability)X
3617(in)X
3709(the)X
3841(limit.)X
576 4194(That)N
761(is,)X
865(inferred)X
1166(probability)X
1577(distribution)X
2008(over)X
2187(the)X
2318(underlying)X
2723(machines)X
3079(converges)X
3457(to)X
3548(a)X
3609(highest)X
3886(pro-)X
576 4354(bability)N
878(for)X
2 f
1011(M)X
1 f
8 s
1094 4371(0)N
2 f
11 s
1164 4354(in)N
1264(the)X
1403(limit)X
1 f
1566(.)X
1641(In)X
1745(other)X
1957(words,)X
2224(after)X
2 f
2416(n)X
1 f
2491(steps)X
2698(for)X
2831(some)X
2 f
3048(n)X
1 f
(,)S
3146(all)X
3267(the)X
3407(machines)X
3772(smaller)X
576 4514(than)N
2 f
750(M)X
1 f
8 s
833 4531(0)N
11 s
894 4514(violate)N
1157(some)X
1365(data)X
1534(pair)X
1693(in)X
2 f
1784(S)X
1 f
(,)S
1872(and)X
2 f
2021(M)X
1 f
8 s
2104 4531(0)N
11 s
2165 4514(is)N
2246(the)X
2376(choice)X
2628(forever)X
2903(after)X
3086(step)X
2 f
3250(n)X
1 f
(.)S
776 4702(\(b\))N
905(It)X
986(is)X
1072(interesting)X
1472(to)X
1568(notice)X
1811(that)X
1971(the)X
2 f
2106(a)X
2177(posteriori)X
1 f
2552(probability)X
2967(decreases)X
3330(monotonically)X
3869(until)X
576 4862(it)N
657(converges)X
1044(to)X
2 f
1144(c)X
1 f
1204(2)X
2 f
8 s
9 f
4827(-)Y
2 f
1283(l)X
1 f
1311(\()X
2 f
1332(p)X
1 f
1369(\))X
2 f
11 s
1404 4862(/)N
1436(d)X
1 f
1511(for)X
2 f
1644(p)X
1 f
1719(the)X
1858(program)X
2187(with)X
2375(which)X
2 f
2621(U)X
1 f
2715(simulates)X
2 f
3080(M)X
1 f
8 s
3163 4879(0)N
11 s
3202 4862(.)N
3255(Smaller)X
3561(machines)X
3925(are)X
576 5022(chosen)N
842(\256rst)X
1001(and)X
1150(then)X
1324(canceled)X
1654(because)X
1954(they)X
2128(violate)X
2391(some)X
2599(data.)X
3 f
776 5210(Predicting)N
1200(extrapolation.)X
1 f
1780(If)X
1870(we)X
2005(want)X
2209(to)X
2311(infer)X
2 f
2517(f)X
1 f
2570(\()X
2 f
2599(s)X
1 f
2640(\),)X
2724(rather)X
2962(than)X
2 f
3154(f)X
1 f
3179(,)X
3234(given)X
3463(the)X
3604(sequence)X
3959(of)X
576 5370(examples)N
2 f
938(S)X
1 f
(,)S
1033(then)X
1214(using)X
1434(formulas)X
1776(\(2\))X
1907(and)X
2062(\(5\),)X
2214(the)X
2350(inferred)X
2656(probability)X
3072(that)X
2 f
3240(f)X
1 f
3293(\()X
2 f
3322(s)X
1 f
3363(\))X
2 f
9 f
3406(=)X
2 f
3468(a)X
1 f
3540(is)X
3627(\(denoting)X
3993(a)X
576 5530(string)N
2 f
799(s)X
1 f
8 s
843 5547(1)N
2 f
11 s
882 5530(s)N
1 f
8 s
926 5547(2)N
11 s
987 5504(.)N
1031(.)X
1075(.)X
2 f
1119 5530(s)N
8 s
1153 5547(n)N
1 f
11 s
1214 5530(as)N
2 f
1309(s)X
1 f
8 s
1353 5547(1:)N
2 f
1403(n)X
1 f
11 s
1442 5530(\):)N
2 f
1016 5831(P)N
1 f
1084(\()X
2 f
1120(f)X
1 f
1173(\()X
2 f
1202(s)X
1 f
1243(\))X
2 f
9 f
1286(=)X
2 f
1348(a)X
1 f
9 f
1420(|)X
2 f
1473(f)X
1 f
1526(\()X
2 f
1555(s)X
8 s
1589 5848(i)N
1 f
11 s
1614 5831(\))N
2 f
9 f
1657(=)X
2 f
1719(a)X
8 s
5848(i)Y
1 f
11 s
1788 5831(,)N
2 f
1831(i)X
9 f
1877(=)X
1 f
1939(1,)X
2012(..,)X
2 f
2085(n)X
1 f
2136(\))X
2 f
9 f
2187(=)X
1 f
16 s
9 f
2444 5918(S)N
2 f
11 s
2534 5892({)N
3 f
2569(m)X
1 f
2642(\()X
2 f
2678(j)X
1 f
2710(\):)X
2 f
2785(f)X
8 s
2815 5909(j)N
1 f
11 s
2840 5892(\()N
2 f
2869(s)X
8 s
2903 5909(i)N
1 f
11 s
2928 5892(\))N
2 f
9 f
2971(=)X
2 f
3033(a)X
8 s
5909(i)Y
1 f
11 s
3102 5892(,)N
2 f
3145(i)X
9 f
3191(=)X
1 f
3253(1,)X
3326(..,)X
2 f
3399(n)X
3450(})X
1 f
16 s
9 f
2279 5778(S)N
2 f
11 s
2369 5752({)N
3 f
2404(m)X
1 f
2477(\()X
2 f
2513(j)X
1 f
2545(\):)X
2 f
2620(f)X
8 s
2650 5769(j)N
1 f
11 s
2675 5752(\()N
2 f
2704(s)X
8 s
2738 5769(i)N
1 f
11 s
2763 5752(\))N
2 f
9 f
2806(=)X
2 f
2868(a)X
8 s
5769(i)Y
1 f
11 s
2937 5752(,)N
2 f
2980(i)X
9 f
3026(=)X
1 f
3088(1,)X
3161(..,)X
2 f
3234(n)X
1 f
(,)S
2 f
3335(f)X
8 s
3365 5769(j)N
1 f
11 s
3390 5752(\()N
2 f
3419(s)X
1 f
3460(\))X
2 f
9 f
3503(=)X
2 f
3565(a)X
3616(})X
1 f
10 f
2265 5805(h)N
2300(hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh)X
1 f
3930 5831(\(9\))N
ep
%%Page: 26 26
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(26)X
2370(-)X
776 736(The)N
945(Gold)X
1153(paradigm)X
1518(of)X
1623(inductive)X
1984(inference)X
2343(in)X
2444(the)X
2584(sense)X
2806(as)X
2911(originally)X
3287(studied)X
3574(by)X
3694(Gold)X
3902(in)X
4003([)X
576 896(Gold)N
776(Language)X
1147(identi\256cation)X
1638(1967)X
1838(])X
1891([)X
1944(Gold)X
2144(1978)X
2344(can)X
2490(be)X
2597(viewed)X
2875(simply)X
3140(as)X
3237(a)X
3299(computable)X
3734(approxi-)X
576 1056(mation)N
874(to)X
995(Equation)X
1366(\(8\).)X
1564(The)X
1753(fundamental)X
2246(idea)X
2446(of)X
2572(the)X
2733(Gold)X
2962(paradigm)X
3348(is)X
3460(the)X
3621(idea)X
3821(called)X
2 f
576 1216(identi\256cation)N
1067(in)X
1160(the)X
1292(limit)X
1 f
1479(and)X
1630(a)X
1692(universal)X
2038(method)X
2326(of)X
2422(implementi)X
2826(ng)X
2937(the)X
3068(identi\256cation)X
3558(in)X
3650(the)X
3781(limit)X
3973(is)X
576 1376(called)N
815(`identi\256cation)X
1339(by)X
1455(enumeration'.)X
1975(These)X
2214(are)X
2350(contained)X
2722(in)X
2820(facts)X
3015(\(a\))X
3141(and)X
3297(\(b\),)X
3450(as)X
3552(a)X
3620(computable)X
576 1536(analogue)N
923(of)X
1025(Solomonoff's)X
1538(approach.)X
1911(We)X
2062(now)X
2242(investigate)X
2654(the)X
2791(correspondence)X
3370(between)X
3691(these)X
3900(two)X
576 1696(basic)N
779(ideas)X
982(in)X
1073(some)X
1281(detail.)X
3 f
776 1884(Identi\256cation)N
1302(in)X
1398(the)X
1537(Limit)X
1 f
1770(views)X
1997(inductive)X
2348(inference)X
2697(as)X
2792(an)X
2897(in\256nite)X
3170(process.)X
3478(Formally,)X
3847(let)X
2 f
3959(M)X
1 f
576 2044(be)N
685(an)X
794(inductive)X
1149(inference)X
1502(method)X
1793(in)X
1888(order)X
2099(to)X
2194(derive)X
2440(some)X
2652(unknown)X
3005(rule)X
2 f
3168(R)X
1 f
3222(.)X
3269(If)X
2 f
3352(M)X
1 f
3450(receives)X
3763(a)X
3827(larger)X
576 2204(and)N
730(larger)X
962(set)X
1088(of)X
1189(examples)X
1550(\(bigger)X
1832(and)X
1987(bigger)X
2240(initial)X
2476(segment)X
2 f
2798(S)X
1 f
(\),)S
2921(a)X
2988(larger)X
3221(and)X
3376(larger)X
3609(sequence)X
3959(of)X
2 f
576 2364(M)N
1 f
649('s)X
749(conjectures)X
1187(is)X
1283(generated,)X
1684(say,)X
2 f
1867(f)X
1 f
8 s
1902 2381(1)N
11 s
1941 2364(,)N
2 f
1977(f)X
1 f
8 s
2012 2381(2)N
11 s
2051 2364(,)N
2 f
2087(f)X
1 f
8 s
2122 2381(3)N
11 s
2161 2364(,)N
2212 2338(.)N
2256(.)X
2300(.)X
2344 2364(.)N
2403(If)X
2498(there)X
2711(is)X
2807(some)X
3030(integer)X
2 f
3312(m)X
1 f
3411(such)X
3608(that)X
2 f
3784(f)X
8 s
3809 2381(m)N
1 f
11 s
3898 2364(is)N
3993(a)X
576 2524(correct)N
842(description)X
1256(of)X
2 f
1351(R)X
1 f
1427(and)X
1576(for)X
1700(all)X
2 f
1811(n)X
1869(>m)X
2194 2764(f)N
8 s
2219 2781(m)N
11 s
9 f
2272 2764(=)N
2 f
2327(f)X
8 s
2352 2781(n)N
1 f
11 s
2391 2764(,)N
576 3004(then)N
2 f
758(M)X
861(identi\256ed)X
1221(R)X
1 f
1306(in)X
1406(the)X
1545(limit.)X
1767(Two)X
1959(facts)X
2156(deserve)X
2455(mentioning:)X
2 f
2914(M)X
1 f
3018(cannot)X
3284(determine)X
3668(whether)X
3982(it)X
576 3164(has)N
723(converged)X
1119(and)X
1276(therefore)X
1622(stop)X
1798(with)X
1984(a)X
2052(correct)X
2325(hypothesis.)X
2 f
2753(M)X
1 f
2855(may)X
3036(be)X
3148(viewed)X
3431(as)X
3533(learning)X
3851(more)X
576 3324(and)N
728(more)X
934(information)X
1377(about)X
1599(the)X
1733(unknown)X
2086(rule)X
2 f
2249(R)X
1 f
2329(and)X
2482(monotonically)X
3019(increasing)X
3407(its)X
3517(approximation)X
576 3484(to)N
2 f
667(R)X
1 f
743(until)X
928(the)X
1058(correct)X
1324(identi\256cation.)X
1857(Gold)X
2055(gave)X
2243(the)X
2373(best)X
2537(explanation)X
2971(to)X
3062(his)X
3187(de\256nition:)X
10 s
776 3656(I)N
826(wish)X
1001(to)X
1087(construct)X
1405(a)X
1465(precise)X
1717(model)X
1941(for)X
2059(the)X
2181(intuitive)X
2471(notion)X
2699("able)X
2890(to)X
2976(speak)X
3183(a)X
3243(language")X
3590(in)X
3676(order)X
3870(to)X
3956(be)X
576 3800(able)N
731(to)X
814(investigate)X
1182(theoretically)X
1604(how)X
1763(it)X
1828(can)X
1961(be)X
2058(achieved)X
2365(arti\256cially.)X
2735(Since)X
2934(we)X
3049(cannot)X
3284(write)X
3470(down)X
3669(the)X
3788(rules)X
3965(of)X
576 3944(English)N
852(which)X
1080(we)X
1206(require)X
1466(one)X
1614(to)X
1708(know)X
1918(before)X
2156(we)X
2282(say)X
2421(he)X
2529(can)X
2673("speak)X
2921(English",)X
3250(an)X
3358(arti\256cial)X
3658(intelligence)X
576 4088(which)N
795(is)X
871(designed)X
1179(to)X
1264(speak)X
1470(English)X
1737(will)X
1884(have)X
2058(to)X
2142(learn)X
2325(its)X
2422(rules)X
2600(from)X
2778(implicit)X
3048(information.)X
3488(That)X
3657(is,)X
3752(its)X
3849(infor-)X
576 4232(mation)N
819(will)X
964(consist)X
1207(of)X
1295(examples)X
1619(of)X
1707(the)X
1826(use)X
1954(of)X
2042(English)X
2307(and/or)X
2533(of)X
2621(an)X
2718(informant)X
3055(who)X
3214(can)X
3347(state)X
3515(whether)X
3796(a)X
3854(given)X
576 4376(usage)N
779(satis\256es)X
1052(certain)X
1291(rules)X
1467(of)X
1554(English,)X
1838(but)X
1960(cannot)X
2194(state)X
2361(these)X
2546(rules)X
2722(explicitly.)X
11 s
798 4522(.)N
842(.)X
886(.)X
10 s
951 4548(A)N
1030(person)X
1265(does)X
1434(not)X
1558(know)X
1758(when)X
1954(he)X
2052(is)X
2127(speaking)X
2434(a)X
2492(language)X
2804(correctly;)X
3134(there)X
3317(is)X
3392(always)X
3637(the)X
3757(possibil-)X
576 4692(ity)N
687(that)X
834(he)X
937(will)X
1087(\256nd)X
1237(that)X
1383(his)X
1502(grammar)X
1818(contains)X
2111(an)X
2213(error.)X
2416(But)X
2557(we)X
2677(can)X
2815(guarantee)X
3154(that)X
3300(a)X
3362(child)X
3548(will)X
3698(eventually)X
576 4836(learn)N
757(a)X
813(natural)X
1056(language,)X
1386(even)X
1558(if)X
1627(it)X
1691(will)X
1835(not)X
1957(know)X
2155(when)X
2349(it)X
2413(is)X
2486(correct.)X
3 f
11 s
776 5024(Identi\256cation)N
1320(by)X
1453(enumeration)X
1 f
1972(is)X
2071(a)X
2150(method)X
2455(to)X
2564(implement)X
2983(identi\256cation)X
3490(in)X
3599(the)X
3747(limit.)X
3978(It)X
576 5184(refers)N
803(to)X
900(the)X
1036(following)X
1407(guessing)X
1743(rule:)X
1933(Enumerate)X
2343(the)X
2479(class)X
2677(of)X
2777(rules)X
2975(in)X
3071(rule)X
3235(space.)X
3479(At)X
3594(step)X
2 f
3763(t)X
1 f
3788(,)X
3837(guess)X
576 5344(the)N
712(unknown)X
1067(rule)X
1232(to)X
1330(be)X
1442(the)X
1579(\256rst)X
1745(rule)X
1911(of)X
2013(the)X
2150(enumeration)X
2620(which)X
2864(agrees)X
3117(with)X
3303(data)X
3479(received)X
3806(so)X
3913(far.)X
576 5504(Formally)N
926(speaking,)X
1287(in)X
1382(our)X
1525(setting,)X
1809(if)X
1889(we)X
2017(have)X
2209(received)X
2533(an)X
2642(initial)X
2876(segment)X
2 f
3195(S)X
1 f
(,)S
3286(then,)X
3485(given)X
2 f
3706(s)X
1 f
3740(,)X
3787(predict)X
576 5664(as)N
679(the)X
817(next)X
999(example)X
1328(\()X
2 f
1357(s)X
1 f
1391(,)X
2 f
1448(f)X
1 f
1501(\()X
2 f
1530(s)X
1 f
1571(\)\))X
1659(for)X
2 f
1798(f)X
1 f
1853(is)X
1942(the)X
2080(\256rst)X
2247(rule)X
2414(that)X
2577(is)X
2666(consistent)X
3049(with)X
2 f
3236(S)X
1 f
(.)S
3354(Now)X
3554(if)X
3638(this)X
3796(can)X
3949(be)X
576 5824(done)N
2 f
795(effectively)X
1 f
1154(,)X
1224(identi\256cation)X
1739(in)X
1855(the)X
2010(limit)X
2226(will)X
2411(be)X
2541(achieved.)X
2923(We)X
3092(say)X
3256(an)X
3386(induction)X
3767(method)X
ep
%%Page: 27 27
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(27)X
2370(-)X
576 736(identi\256es)N
928(a)X
995(rule)X
1160(correctly)X
1501(in)X
2 f
1599(k)X
1 f
1667(steps)X
1872(if)X
1955(it)X
2034(will)X
2201(never)X
2425(produce)X
2737(wrong)X
2990(hypothesis)X
3396(starting)X
3690(from)X
3890(step)X
2 f
576 896(k)N
8 s
625 861(*)N
1 f
11 s
664 896(.)N
735(Let)X
2 f
880(G)X
1 f
970(and)X
2 f
1124(G)X
9 f
1187(\242)X
1 f
1236(be)X
1346(two)X
1505(guessing)X
1840(methods.)X
2 f
2210(G)X
1 f
2300(will)X
2465(be)X
2575(said)X
2744(to)X
2840(be)X
2 f
2950(uniformly)X
3320(faster)X
1 f
3549(than)X
2 f
3729(G)X
9 f
3792(\242)X
1 f
3842(if)X
3924(the)X
576 1056(following)N
943(two)X
1099(conditions)X
1490(hold:)X
1695(\(1\))X
1820(Given)X
2058(any)X
2 f
2208(R)X
1 f
2285(from)X
2479(the)X
2610(rule)X
2770(space,)X
2 f
3010(G)X
1 f
3096(will)X
3257(identify)X
2 f
3555(R)X
1 f
3632(correctly)X
3968(at)X
576 1216(least)N
764(as)X
863(soon)X
1055(as)X
2 f
1154(G)X
9 f
1217(\242)X
1 f
1239(,)X
1288(expressed)X
1661(in)X
1757(the)X
1892(number)X
2188(of)X
2288(examples)X
2648(needed,)X
2946(for)X
3075(all)X
3191(sequences)X
3574(of)X
3674(examples;)X
576 1376(and)N
727(\(2\))X
853(for)X
979(some)X
2 f
1189(R)X
1 f
1243(,)X
1289(some)X
1499(sequence)X
1845(of)X
1942(examples,)X
2 f
2321(G)X
1 f
2408(will)X
2570(identify)X
2 f
2868(R)X
1 f
2945(sooner)X
3202(than)X
2 f
3377(G)X
9 f
3440(\242)X
1 f
3462(.)X
3507(Say)X
3662(a)X
3724(guessing)X
576 1536(method)N
2 f
874(G)X
1 f
970(is)X
2 f
1062(optimal)X
1 f
1366(if)X
1454(for)X
1590(any)X
1751(other)X
1966(guessing)X
2308(method)X
2 f
2607(G)X
9 f
2670(\242)X
1 f
2726(there)X
2936(is)X
3029(a)X
3102(constant)X
2 f
3430(c)X
1 f
3503(such)X
3698(that:)X
3890(if)X
2 f
3978(R)X
1 f
576 1696(appears)N
867(to)X
959(be)X
1065(the)X
2 f
1196(i)X
1 f
1221(th)X
1313(rule)X
1473(in)X
1565(the)X
1696(enumeration)X
2160(of)X
2 f
2256(G)X
9 f
2319(\242)X
1 f
2341(,)X
2386(then)X
2561(it)X
2634(appears)X
2925(no)X
3036(later)X
3216(than)X
2 f
3391(ci)X
1 f
3478(in)X
3570(the)X
3700(enumera-)X
576 1856(tion)N
756(of)X
2 f
871(G)X
1 f
934(.)X
1020(It)X
1116(is)X
1217(easy)X
1415(to)X
1526(prove)X
1768(that)X
1943(the)X
2093(identi\256cation-by-enumerati)X
3059(on)X
3189(method)X
3496(will)X
3676(identify)X
3993(a)X
576 2016(hypothesis)N
982(in)X
1080(the)X
1217(limit)X
1415(if)X
1498(this)X
1655(hypothesis)X
2061(can)X
2212(be)X
2324(identi\256ed)X
2687(in)X
2785(the)X
2922(limit)X
3120(at)X
3213(all.)X
3353(Further)X
3640(if)X
2 f
3722(G)X
1 f
8 s
3795 2033(0)N
11 s
3862 2016(is)N
3949(an)X
576 2176(identi\256cation-by-enumerati)N
1542(on)X
1658(guessing)X
1994(rule,)X
2181(then)X
2361(there)X
2566(is)X
2654(no)X
2771(guessing)X
3108(rule)X
3274(uniformly)X
3656(faster)X
3880(than)X
2 f
576 2336(G)N
1 f
8 s
649 2353(0)N
11 s
688 2336(.)N
754(But)X
904(only)X
1083(the)X
1213(Solomonoff)X
1656(procedure)X
2029(is)X
2110(optimal.)X
2447(Indeed,)X
3 f
776 2524(Theorem)N
1147(4)X
1 f
(.)S
1242(\(a\))X
2 f
1369(Identi\256cation-by-enumeration)X
2456(is)X
2545(a)X
2619(computable)X
3060(approximation)X
3609(to)X
3708(inductive)X
576 2684(inference)N
931(\(Solomonoff's)X
1452(inference\))X
1836(associated)X
2235(with)X
2415(Formula)X
2750(\(8\).)X
2900(\(b\))X
3028(Neither)X
3319(method)X
3604(is)X
3689(uniformly)X
576 2844(faster)N
801(than)X
982(the)X
1114(other.)X
1346(\(c\))X
1467(Solomonoff)X
1893(procedure)X
2278(is)X
2361(optimal,)X
2678(while)X
2895(identi\256cation-by-enumeration)X
3973(is)X
576 3004(not.)N
776 3192(Proof)N
991(.)X
1 f
1063(\(a\))X
1188(An)X
1324(effective)X
1661(enumeration)X
2131(for)X
2262(the)X
2399(identi\256cation-by-enumerati)X
3365(on)X
3482(method,)X
3798(can)X
3949(be)X
576 3352(viewed)N
863(as)X
969(a)X
1041(computable)X
1486(approximation)X
2034(to)X
2135(Solomonoff's)X
2651(procedure)X
3034(according)X
3413(to)X
3514(formula)X
3825(\(8\))X
3959(as)X
576 3512(follows.)N
915(Let)X
1064(the)X
1203(effective)X
1543(enumeration)X
2016(of)X
2121(the)X
2261(rule)X
2430(space)X
2657(be:)X
2 f
2797(R)X
1 f
8 s
2861 3529(1)N
11 s
2900 3512(,)N
2 f
2929(R)X
1 f
8 s
2993 3529(2)N
11 s
3032 3512(,)N
2 f
3061(R)X
1 f
8 s
3125 3529(3)N
11 s
3186 3486(.)N
3230(.)X
3274(.)X
3318 3512(.)N
3372(Convert)X
3688(this)X
3848(to)X
3949(an)X
576 3672(effective)N
909(pre\256x-free)X
1304(description)X
1720(of)X
1817(each)X
2002(rule)X
2 f
2163(R)X
8 s
2217 3689(i)N
1 f
11 s
2266 3672(in)N
2359(the)X
2491(rule)X
2652(space.)X
2915(For)X
3061(instance,)X
3396(if)X
2 f
3474(x)X
9 f
3534(=)X
2 f
3596(x)X
1 f
8 s
3645 3689(1)N
11 s
3684 3672(,)N
2 f
3713(...)X
1 f
(,)S
2 f
3808(x)X
8 s
3847 3689(n)N
1 f
11 s
3910 3672(is)N
3993(a)X
576 3836(binary)N
828(string,)X
1078(then)X
2 f
1257(x)X
1 f
10 f
1257 3777(h)N
2 f
9 f
1317 3836(=)N
2 f
1379(x)X
1 f
8 s
1428 3853(1)N
11 s
1467 3836(0)N
2 f
(x)S
1 f
8 s
1560 3853(2)N
11 s
1599 3836(0...0)N
2 f
(x)S
8 s
1792 3853(n)N
1 f
11 s
1831 3836(1)N
1902(is)X
1988(a)X
2 f
2055(pre\256x-code)X
1 f
2478(for)X
2608(the)X
2 f
2744(x)X
1 f
2783('s.)X
2896(Similarly,)X
2 f
3276(x)X
1 f
9 f
3322(\242)X
2 f
9 f
3358(=)X
2 f
3420(l)X
1 f
3459(\()X
2 f
3488(x)X
1 f
3534(\))X
10 f
3428 3755(h)N
3431(hhh)X
2 f
3577 3836(x)N
1 f
3644(is)X
3731(a)X
3798(pre\256x-)X
576 3996(code.)N
786(Note)X
979(that)X
2 f
1134(l)X
1 f
1173(\()X
2 f
1202(x)X
1 f
9 f
1248(\242)X
1 f
1270(\))X
2 f
9 f
1313(=)X
2 f
1375(l)X
1 f
1414(\()X
2 f
1443(x)X
1 f
1489(\))X
2 f
9 f
1532(+)X
1 f
1594(2)X
1652(log)X
2 f
1779(l)X
1 f
1818(\()X
2 f
1847(x)X
1 f
1893(\).)X
1966(We)X
2110(encode)X
2381(each)X
2564(rule)X
2 f
2723(R)X
8 s
2777 4013(i)N
1 f
11 s
2824 3996(\(a)N
2914(binary)X
3161(string\))X
3413(as)X
2 f
3508(p)X
3573(i)X
1 f
9 f
3605(\242)X
1 f
3627(,)X
3671(where)X
2 f
3907(p)X
1 f
3973(is)X
576 4156(a)N
641(\(pre\256x\))X
930(program)X
1254(that)X
1413(enumerates)X
1840(the)X
1974(rule)X
2137(space.)X
2402(The)X
2566(resulting)X
2902(code)X
3095(for)X
3224(the)X
2 f
3359(R)X
8 s
3413 4173(i)N
1 f
11 s
3438 4156('s)N
3528(is)X
3614(an)X
3724(effective)X
576 4316(pre\256x-code.)N
1042(Denoting)X
1392(the)X
1522(length)X
1765(of)X
1860(the)X
1990(description)X
2404(of)X
2 f
2499(R)X
8 s
2553 4333(i)N
1 f
11 s
2600 4316(by)N
9 f
2724(|)X
2 f
2756(R)X
8 s
2810 4333(i)N
1 f
11 s
9 f
2849 4316(|)N
1 f
2881(,)X
2925(we)X
3049(have:)X
576 4504(\(i\))N
776(if)X
2 f
852(i)X
884(<)X
950(j)X
1 f
975(,)X
1019(then)X
9 f
1207(|)X
2 f
1239(R)X
8 s
1293 4521(i)N
1 f
11 s
9 f
1332 4504(|)N
1378(\243)X
1454(|)X
2 f
1486(R)X
8 s
1545 4521(j)N
1 f
11 s
9 f
1584 4504(|)N
1 f
1616(;)X
1663(and)X
576 4698(\(ii\))N
2 f
8 s
805 4788(i)N
1 f
16 s
9 f
776 4724(S)N
11 s
1 f
852 4698(2)N
2 f
8 s
9 f
4650(-)Y
1 f
9 f
941(|)X
2 f
964(R)X
5 s
1003 4662(i)N
1 f
8 s
9 f
1029 4650(|)N
11 s
1059 4698(\243)N
1 f
1107(1)X
1173(\(by)X
1312(Kraft's)X
1582(inequality\).)X
576 4964(Assign)N
2 f
847(a)X
918(priori)X
1 f
1151(probability)X
2 f
1566(P)X
1 f
1634(\()X
2 f
1663(R)X
8 s
1717 4981(i)N
1 f
11 s
1742 4964(\))N
2 f
9 f
1785(=)X
1 f
1847(2)X
2 f
8 s
9 f
4916(-)Y
1 f
9 f
1946(|)X
2 f
1969(R)X
5 s
2008 4928(i)N
1 f
8 s
9 f
2034 4916(|)N
11 s
1 f
2091 4964(to)N
2187(rule)X
2 f
2351(R)X
8 s
2405 4981(i)N
1 f
11 s
2430 4964(,)N
2 f
2479(i)X
9 f
2525(=)X
1 f
2573(1,)X
2660(2,)X
2733(....)X
2848(\(This)X
3062(is)X
3149(possible)X
3466(because)X
3772(of)X
3873(\(ii\).\))X
576 5124(Using)N
849(Formula)X
1211(\(8\))X
1376(with)X
2 f
1596(P)X
1 f
1664(\()X
2 f
1693(R)X
8 s
1747 5141(i)N
1 f
11 s
1772 5124(\))N
1864(instead)X
2177(of)X
3 f
2313(m)X
1 f
2386(\()X
2 f
2415(i)X
1 f
2447(\))X
2538(yields)X
2811(a)X
2912(computable)X
3386(approximation)X
3963(to)X
576 5284(Solomonoff's)N
1123(inductive)X
1515(inference)X
1905(procedure.)X
2363(Formula)X
2725(\(8\))X
2890(chooses)X
3231(the)X
3402(shortest)X
3739(encoded)X
9 s
10 f
576 5372(hhhhhhhhhhhhhhhh)N
1 f
576 5488(*)N
633(Although)X
926(here)X
1071(we)X
1176(treat)X
1325(only)X
1474(the)X
1583(case)X
1728(when)X
1905(the)X
2014(procedure)X
2323(converges)X
2636(to)X
2713(one)X
2838(single)X
3031(rule,)X
3182(this)X
3307(de\256nition)X
3604(allows)X
3814(that)X
3944(the)X
576 5576(procedure)N
887(vacillate)X
1154(between)X
1417(the)X
1528(correct)X
1751(rules.)X
1932(Such)X
2098(de\256nition)X
2396(is)X
2466(needed)X
2692(when,)X
2888(say,)X
3024(function)X
2 f
3286(f)X
1 f
3328(is)X
3398(not)X
3512(computable.)X
3888(Such)X
576 5664(research)N
834(has)X
948(been)X
1102(initiated)X
1356(by)X
1446(J.)X
1510(Case)X
1668([)X
1710(case)X
1852(power)X
2050(of)X
2128(vacillation)X
ep
%%Page: 28 28
bp
9 s 0 xH 0 xS 1 f
11 s
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(28)X
2370(-)X
576 736(consistent)N
961(rule)X
1130(which)X
1377(coincides)X
1742(with)X
1931(the)X
2071(\256rst)X
2240(consistent)X
2625(rule)X
2795(in)X
2897(the)X
3038(effective)X
3379(enumeration.)X
3875(This)X
576 896(shows)N
826(that)X
990(identi\256cation)X
1488(by)X
1607(enumeration)X
2078(can)X
2230(be)X
2343(formulated)X
2760(as)X
2863(an)X
2976(computable)X
3418(approximation)X
3963(to)X
576 1056(Solomonoff's)N
1082(procedure.)X
776 1244(We)N
925(now)X
1103(show)X
1315(that)X
1476(neither)X
1749(method)X
2042(is)X
2129(uniformly)X
2510(faster)X
2733(than)X
2913(the)X
3049(other.)X
3302(Let)X
2 f
3448(G)X
1 f
8 s
3521 1261(1)N
11 s
3560 1244(,)N
2 f
3603(G)X
1 f
8 s
3676 1261(2)N
11 s
3715 1244(,)N
2 f
3744(...)X
1 f
3838(be)X
3949(an)X
576 1404(effective)N
920(enumeration)X
1397(of)X
1506(the)X
1650(hypotheses)X
2077(space)X
2307(by)X
2430(a)X
2504(Gold)X
2715(procedure,)X
3123(and)X
3285(let)X
2 f
3409(H)X
1 f
8 s
3482 1421(1)N
11 s
3521 1404(,)N
2 f
3564(H)X
1 f
8 s
3637 1421(2)N
11 s
3676 1404(,)N
2 f
3705(...)X
1 f
3806(be)X
3924(the)X
576 1564(\(noneffective\))N
1101(enumeration)X
1569(of)X
1669(the)X
1804(hypotheses)X
2222(space)X
2444(by)X
2559(decreasing)X
2 f
2963(a)X
3035(priori)X
1 f
3269(probability)X
3685(according)X
576 1724(to)N
672(Solomonoff.)X
1142(In)X
1242(other)X
1450(words,)X
2 f
1713(K)X
1 f
1779(\()X
2 f
1808(H)X
1 f
8 s
1881 1741(1)N
11 s
1920 1724(\))N
9 f
1963(\243)X
2 f
2025(K)X
1 f
2098(\()X
2 f
2127(H)X
1 f
8 s
2200 1741(2)N
11 s
2239 1724(\))N
9 f
2282(\243)X
1 f
2352 1698(.)N
2396(.)X
2440(.)X
2484 1724(.)N
2555(In)X
2655(both)X
2839(cases)X
3051(we)X
3180(deal)X
3353(with)X
3536(identi\256cation-)X
576 1884(by-enumeration,)N
1178(so)X
1278(it)X
1350(is)X
1431(known)X
1692(that)X
1847(there)X
2045(is)X
2126(no)X
2236(guessing)X
2566(rule)X
2725(uniformly)X
3100(faster)X
3317(than)X
3491(either)X
3714(of)X
3809(them.)X
776 2072(To)N
906(prove)X
1138(\(c\),)X
1290(let)X
1412(an)X
1528(arbitrary)X
1864(procedure)X
2 f
2248(G)X
1 f
2344(using)X
2568(identi\256cation-by-enumerati)X
3534(on)X
3655(effectively)X
576 2232(enumerates)N
1012(rules)X
1218(in)X
1322(our)X
1474(rule)X
1646(space)X
1875(as)X
2 f
1982(R)X
1 f
8 s
2046 2249(1)N
11 s
2085 2232(,)N
2 f
2114(R)X
1 f
8 s
2178 2249(2)N
11 s
2217 2232(,)N
2268 2206(.)N
2312(.)X
2356(.)X
2400 2232(.)N
2478(Then)X
2693(obviously)X
2 f
3075(K)X
1 f
3148(\()X
2 f
3177(R)X
8 s
3231 2249(i)N
1 f
11 s
3256 2232(\))N
2 f
9 f
3285(=)X
2 f
3333(K)X
1 f
3406(\()X
2 f
3435(i)X
1 f
3467(\))X
2 f
9 f
3496(+)X
2 f
3544(c)X
8 s
3583 2249(G)N
1 f
11 s
3636 2232(,)N
3692(where)X
2 f
3940(c)X
8 s
3979 2249(G)N
1 f
11 s
576 2427(depends)N
888(on)X
2 f
1000(G)X
1 f
1063(.)X
1109(Hence)X
3 f
1357(m)X
1 f
1430(\()X
2 f
1459(R)X
8 s
1513 2444(i)N
1 f
11 s
1538 2427(\))N
1591(is)X
1674(approximately)X
2208(at)X
2296(least)X
2 f
2504 2488(c)N
1 f
2550 2462(.)N
2 f
2488(i)Y
1 f
2529 2375(1)N
10 f
2491 2401(h)N
2523(hh)X
1 f
2643 2427(for)N
2769(some)X
2 f
2979(c)X
1 f
3018(.)X
3087(Hence)X
3336(the)X
3469(number)X
3763(of)X
3861(rules)X
576 2631(that)N
731(have)X
919(probability)X
1329(greater)X
1595(than)X
1769(this)X
1919(is)X
2000(at)X
2086(most)X
2 f
2280(c)X
1 f
2326 2605(.)N
2 f
2631(i)Y
1 f
2373(.)X
10 f
2439(`)X
3 f
776 2819(Remark.)N
1 f
1158(What)X
1380(about)X
1607(non-uniform)X
2083(speed)X
2314(comparison?)X
2817(In)X
2922(case)X
3105(the)X
3245(particular)X
3615(rule)X
2 f
3791(f)X
1 f
3848(to)X
3949(be)X
576 2979(inferred)N
889(is)X
983(suf\256ciently)X
2 f
1415(simple)X
1 f
1680(\(has)X
1861(low)X
2028(Kolmogorov)X
2513(complexity\))X
2975(then)X
3162(Solomonoff's)X
3681(procedure)X
576 3139(can)N
733(be)X
851(much)X
1082(faster)X
1312(than)X
1499(Gold's)X
1773(enumeration.)X
2293(Let)X
2 f
2453(f)X
1 f
2513(be)X
2632(the)X
2776(function)X
3106(we)X
3244(want)X
3451(to)X
3556(infer,)X
3780(and)X
3943(let)X
2 f
583 3299(f)N
9 f
629(=)X
2 f
698(f)X
8 s
723 3316(m)N
1 f
11 s
776 3299(,)N
827(with)X
2 f
1013(m)X
1 f
1105(minimal,)X
1452(in)X
1550(Gold's)X
1817(enumeration)X
2286(order.)X
2543(Let)X
2689(also)X
2 f
2888(f)X
9 f
2934(=)X
2 f
3003(f)X
8 s
3028 3316(n)N
1 f
11 s
3067 3299(,)N
3117(for)X
2 f
3247(n)X
1 f
3319(with)X
2 f
3504(K)X
1 f
3577(\()X
2 f
3606(n)X
1 f
3657(\))X
3714(minimal.)X
576 3459(To)N
725(infer)X
942(the)X
1101(correct)X
2 f
1403(f)X
1 f
1428(,)X
1501(in)X
1621(Gold's)X
1911(approach)X
2284(we)X
2438(must)X
2662(eliminate)X
3044(all)X
2 f
3192(f)X
8 s
3217 3476(k)N
1 f
11 s
3304 3459(with)N
2 f
3513(k)X
3573(<)X
3646(m)X
1 f
3709(.)X
3783(But)X
3963(in)X
576 3619(Solomonoff's)N
1082(approach,)X
1448(we)X
1572(only)X
1751(need)X
1939(to)X
2030(eliminate)X
2382(all)X
2 f
2500(f)X
8 s
2525 3636(k)N
1 f
11 s
2582 3619(with)N
2 f
2761(K)X
1 f
2834(\()X
2 f
2863(k)X
1 f
2909(\))X
2 f
2952(<)X
3025(K)X
1 f
3098(\()X
2 f
3127(n)X
1 f
3178(\).)X
3251(Now)X
3443(necessarily)X
3856(there)X
576 3779(are)N
714(many)X
2 f
948(f)X
1 f
973('s)X
1067(that)X
1232(are)X
1371(`simple')X
1697(in)X
1798(the)X
1938(sense)X
2160(that)X
2 f
2325(K)X
1 f
2398(\()X
2 f
2427(n)X
1 f
2478(\))X
2521(<)X
2545(<)X
2 f
2609(l)X
1 f
2648(\()X
2 f
2677(m)X
1 f
2747(\),)X
2830(for)X
2964(which)X
3211(e.g.)X
3370(Solomonoff's)X
3886(pro-)X
576 3939(cedure)N
832(works)X
1068(much)X
1286(\(sometimes)X
1715(noncomputably\))X
2315(faster)X
2532(than)X
2706(Gold's)X
2967(method.)X
776 4127(The)N
943(following)X
1316(theorem)X
1635(sets)X
1797(limits)X
2030(on)X
2148(the)X
2286(number)X
2585(of)X
2688(examples)X
3051(needed)X
3330(to)X
3429(infer)X
3625(a)X
3694(particular)X
576 4287(function)N
2 f
899(f)X
1 f
924(.)X
3 f
776 4475(Theorem)N
1146(5.)X
2 f
1262(Let)X
1410(f)X
1 f
8 s
1445 4492(1)N
11 s
1484 4475(,)N
2 f
1534(f)X
1 f
8 s
1569 4492(2)N
11 s
1608 4475(,)N
2 f
1637(...)X
1731(be)X
1842(an)X
1958(effective)X
2281(enumeration)X
2754(of)X
2852(the)X
2989(rule)X
3160(space.)X
3411(Suppose)X
3733(we)X
3860(want)X
576 4635(to)N
671(infer)X
871(f)X
9 f
917(=)X
2 f
986(f)X
8 s
1011 4652(i)N
11 s
1036 4635(,)N
1084(with)X
1263(i)X
1314(minimal,)X
1651(from)X
1843(a)X
1913(set)X
2037(of)X
2132(n)X
2202(examples)X
2555(S)X
2625(as)X
2729(in)X
2824(\(7\).)X
2974(Let)X
3113(c)X
3178(be)X
3287(an)X
3400(appropriate)X
3846(large)X
576 4795(enough)N
857(constant.)X
576 4955(\(a\))N
704(If)X
784(K)X
1 f
857(\()X
2 f
886(i)X
1 f
918(\))X
2 f
961(>)X
1034(K)X
1 f
1107(\()X
2 f
1143(f)X
1 f
1196(\()X
2 f
1225(s)X
1 f
8 s
1269 4972(1)N
11 s
1308 4955(\)...)N
2 f
1410(f)X
1 f
1463(\()X
2 f
1492(s)X
8 s
1526 4972(n)N
1 f
11 s
1565 4955(\))N
9 f
1608(|)X
2 f
1654(s)X
1 f
8 s
1698 4972(1)N
2 f
11 s
1737 4955(...s)N
8 s
1837 4972(n)N
1 f
11 s
1876 4955(\))N
2 f
9 f
1919(-)X
2 f
1981(c,)X
2068(then)X
2246(it)X
2322(is)X
2407(impossible)X
2810(by)X
2919(any)X
3072(effective)X
3393(deterministic)X
3881(pro-)X
576 5115(cedure)N
837(to)X
928(infer)X
1124(f)X
1171(correctly.)X
576 5275(\(b\))N
710(If)X
796(we)X
926(can)X
1086(infer)X
1293(f)X
1351(correctly)X
1702(by)X
1818(computable)X
2262(approximation)X
2814(to)X
2916(Solomonoff's)X
3414(method)X
3706(\(8\))X
3841(using)X
576 5435(only)N
750(S,)X
838(and)X
992(c)X
1053(extra)X
1256(bits)X
1406(of)X
1497(information,)X
1958(then)X
2132(K)X
1 f
2205(\()X
2 f
2234(i)X
1 f
9 f
2287(|)X
2 f
2333(S)X
1 f
2384(\))X
2 f
9 f
2427(\243)X
2 f
2489(c.)X
576 5595(\(c\))N
695(If)X
771(K)X
1 f
844(\()X
2 f
873(i)X
1 f
9 f
926(|)X
2 f
972(S)X
1 f
1023(\))X
2 f
9 f
1066(\243)X
2 f
1128(c)X
1189(then)X
1363(we)X
1483(can)X
1632(compute)X
1959(f)X
8 s
1984 5612(i)N
11 s
2031 5595(from)N
2219(S)X
2285(and)X
2439(c)X
2500(bits)X
2650(extra)X
2853(information.)X
776 5783(Proof.)N
1 f
1045(\(a\))X
1166(Otherwise)X
1551(we)X
1677(would)X
1921(be)X
2028(able)X
2199(to)X
2292(compute)X
2 f
2620(i)X
1 f
2669(from)X
2864(a)X
2928(program)X
3251(of)X
3349(length)X
3595(signi\256cantly)X
ep
%%Page: 29 29
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(29)X
2370(-)X
576 736(shorter)N
842(than)X
2 f
1016(K)X
1 f
1089(\()X
2 f
1118(i)X
1 f
1150(\):)X
1226(contradiction.)X
1758(Items)X
1976(\(b\))X
2100(and)X
2249(\(c\))X
2368(are)X
2497(obvious.)X
10 f
2820(`)X
1 f
776 924(There)N
1008(is)X
1094(an)X
1204(enormous)X
1578(amount)X
1870(of)X
1970(research)X
2289(in)X
2385(the)X
2520(area)X
2693(under)X
2920(the)X
3055(title)X
3222(of)X
3323(Gold)X
3527(paradigm.)X
3910(We)X
576 1084(refer)N
777(the)X
921(readers)X
1210(to)X
1315(the)X
1458(articles)X
1748([)X
1812(Angluin)X
2136(Smith)X
2383(])X
2447([)X
2511(Case)X
2717(Smith)X
2964(and)X
3126(the)X
3269(book)X
3480([)X
3544(Osherson)X
3910(We)X
576 1244(present)N
853(three)X
1052(examples)X
1408(of)X
1504(reasoning)X
1869(by)X
1980(means)X
2228(of)X
2324(Gold's)X
2587(paradigm)X
2944(in)X
3037(order)X
3246(to)X
3339(give)X
3515(a)X
3578(\257avor)X
3807(of)X
3904(this)X
576 1404(research)N
890(direction.)X
3 f
776 1592(Example.)N
1 f
1131([Gold,)X
1380(1967])X
1607(We)X
1751(can)X
1895(learn)X
2093(a)X
2154(function)X
2470(in)X
2561(the)X
2691(set)X
2811(of)X
2906(primitive)X
3253(recursive)X
3597(functions.)X
2 f
776 1780(Proof.)N
1 f
1024(Effectively)X
1441(enumerate)X
1833(the)X
1966(set)X
2089(of)X
2187(all)X
2301(primitive)X
2651(recursive)X
2999(functions)X
3353(by)X
9 f
3467(y)X
8 s
1 f
3527 1797(1)N
11 s
3566 1780(,)N
9 f
3595(y)X
8 s
1 f
3655 1797(2)N
11 s
3694 1780(,)N
3745 1754(.)N
3789(.)X
3833(.)X
3877 1780(.)N
3925(On)X
576 1940(any)N
726(initial)X
957(input)X
1161(segment)X
1477(\()X
2 f
1506(x)X
1 f
8 s
1555 1957(1)N
11 s
1594 1940(,)N
2 f
1623(y)X
1 f
8 s
1672 1957(1)N
11 s
1711 1940(\))N
1762 1914(.)N
1806(.)X
1850(.)X
1894 1940(\()N
2 f
1923(x)X
8 s
1962 1957(k)N
1 f
11 s
1997 1940(,)N
2 f
2026(y)X
8 s
2065 1957(k)N
1 f
11 s
2100 1940(\),)N
2173(our)X
2312(inference)X
2661(machine)X
2982(just)X
3132(prints)X
3355(the)X
3485(least)X
2 f
3669(i)X
1 f
3716(such)X
3899(that)X
9 f
576 2100(y)N
2 f
8 s
636 2117(i)N
1 f
11 s
683 2100(is)N
764(consistent)X
1139(with)X
1318(the)X
1448(input,)X
2 f
1674(i.e.)X
1 f
1782(,)X
9 f
1833(y)X
2 f
8 s
1893 2117(i)N
1 f
11 s
1918 2100(\()N
2 f
1947(x)X
8 s
1986 2117(k)N
1 f
11 s
2021 2100(\))N
2 f
9 f
2050(=)X
2 f
2098(y)X
8 s
2137 2117(k)N
1 f
11 s
2194 2100(for)N
2 f
2318(k)X
9 f
2371(=)X
1 f
2419(1,)X
2514 2074(.)N
2558(.)X
2602(.)X
2646 2100(,)N
2 f
2675(n)X
1 f
(.)S
3 f
776 2288(Example.)N
1 f
1153([Gold,)X
1402(1967])X
1629(We)X
1773(cannot)X
2030(learn)X
2228(in)X
2319(general)X
2600(a)X
2661(function)X
2977(in)X
3069(the)X
3200(set)X
3321(of)X
3417(all)X
3529(total)X
3710(recursive)X
576 2448(functions.)N
2 f
776 2636(Proof.)N
1 f
1022(By)X
1148(diagonalization.)X
1738(Suppose)X
2 f
2059(M)X
1 f
2155(can)X
2300(identify)X
2598(all)X
2711(recursive)X
3057(functions.)X
3431(But)X
3583(then)X
3759(one)X
3910(can)X
576 2796(de\256ne)N
815(a)X
878(recursive)X
1224(function)X
2 f
1549(f)X
1 f
1598(so)X
1700(that)X
1857(the)X
1989(guesses)X
2281(of)X
2 f
2378(M)X
1 f
2475(will)X
2637(be)X
2744(wrong)X
2992(on)X
2 f
3111(f)X
1 f
3160(in\256nitely)X
3504(often.)X
3731(We)X
3876(con-)X
576 2956(struct)N
2 f
807(f)X
1 f
860(by)X
976(simply)X
1245(simulating)X
2 f
1647(M)X
1 f
1720(.)X
1770(Let)X
2 f
1923(f)X
1 f
1976(\(0\))X
2 f
9 f
2078(=)X
1 f
2126(0,)X
2220(Suppose)X
2546(the)X
2682(value)X
2902(of)X
2 f
3011(f)X
1 f
3064(\(0\),)X
2 f
3202(f)X
1 f
3255(\(1\),)X
3408 2930(.)N
3452(.)X
3496(.)X
3540 2956(,)N
2 f
3576(f)X
1 f
3629(\()X
2 f
3658(n)X
9 f
3716(-)X
1 f
3764(1\))X
3866(have)X
576 3116(been)N
776(constructed.)X
1238(On)X
1379(input)X
2 f
1595(n)X
1 f
(,)S
1695(simulate)X
2 f
2029(M)X
1 f
2136(on)X
2258(initial)X
2500(input)X
2 f
2722(f)X
1 f
2775(\(0\),)X
2 f
2913(f)X
1 f
2966(\(1\),)X
3119 3090(.)N
3163(.)X
3207(.)X
3251 3116(,)N
2 f
3287(f)X
1 f
3340(\()X
2 f
3369(n)X
9 f
3427(-)X
1 f
3475(1\).)X
3603(Then)X
3817(de\256ne)X
2 f
583 3276(f)N
1 f
636(\()X
2 f
665(n)X
1 f
716(\))X
767(equal)X
980(1)X
1046(plus)X
1215(the)X
1345(guess)X
1562(of)X
2 f
1657(M)X
1 f
1752(\(modulo)X
2073(2\).)X
2190(So)X
2 f
2305(M)X
1 f
2400(never)X
2617(guesses)X
2 f
2914(f)X
1 f
2961(correctly.)X
10 f
3318(`)X
3 f
776 3464(Example.)N
1 f
1167(One)X
1349(of)X
1458(the)X
1602(\256rst)X
1775(studied)X
2066(problem)X
2396(was)X
2568(extrapolating)X
3070(a)X
3145(sequence.)X
3525(A)X
3624(machine)X
2 f
3959(M)X
576 3624(extrapolates)N
1 f
1038(a)X
1103(sequence)X
2 f
1458(f)X
1 f
1511(\(1\),)X
2 f
1649(f)X
1 f
1702(\(2\),)X
1855 3598(.)N
1899(.)X
1943(.)X
2013 3624(as)N
2112(follows.)X
2424(It)X
2504(makes)X
2755(an)X
2864(initial)X
3098(guess)X
2 f
3325(f)X
9 f
3364(\242)X
1 f
3386(\(0\).)X
3535(Then)X
3741(it)X
3816(inputs)X
576 3784(the)N
709(real)X
2 f
873(f)X
1 f
926(\(0\).)X
1075(At)X
1188(step)X
2 f
1355(i)X
1 f
1380(,)X
1427(based)X
1652(on)X
1765(previous)X
2093(inputs)X
2 f
2341(f)X
1 f
2394(\(1\),)X
2 f
2532(f)X
1 f
2585(\(2\),)X
2738 3758(.)N
2782(.)X
2826(.)X
2870 3784(,)N
2 f
2906(f)X
1 f
2959(\()X
2 f
2988(i)X
9 f
3027(-)X
1 f
3075(1\),)X
3196(it)X
3272(guesses)X
2 f
3573(f)X
9 f
3612(\242)X
1 f
3634(\()X
2 f
3663(i)X
1 f
3695(\).)X
3772(If)X
3856(there)X
576 3944(is)N
660(a)X
2 f
724(i)X
1 f
8 s
759 3961(0)N
11 s
823 3944(such)N
1009(that)X
1167(for)X
1294(all)X
2 f
1407(i)X
1446(>i)X
1 f
8 s
1540 3961(0)N
2 f
11 s
1610 3944(f)N
9 f
1649(\242)X
1 f
1671(\()X
2 f
1700(i)X
1 f
1732(\))X
2 f
9 f
1761(=)X
2 f
1816(f)X
1 f
1869(\()X
2 f
1898(i)X
1 f
1930(\),)X
2005(then)X
2181(we)X
2307(say)X
2 f
2448(M)X
2545(extrapolates)X
3012(f)X
1 f
3037(.)X
3083(Bringing)X
3421(everything)X
3822(in)X
3915(our)X
576 4104(setting,)N
865(the)X
1004(initial)X
1243(segment)X
1568(before)X
1823(step)X
2 f
1996(i)X
1 f
2053(is)X
2144(a)X
2215(sequence)X
2569(of)X
2674(pairs)X
2877(\(1,)X
2 f
2986(f)X
1 f
3039(\(1\)\))X
3184(\(2,)X
2 f
3293(f)X
1 f
3346(\(2\)\)...\()X
2 f
3572(i)X
9 f
3611(-)X
1 f
3659(1,)X
2 f
3739(f)X
1 f
3792(\()X
2 f
3821(i)X
9 f
3860(-)X
1 f
3908(1\)\),)X
576 4264(and)N
2 f
730(M)X
1 f
830(extrapolates)X
1283(with)X
1467(the)X
1602(pair)X
1766(\()X
2 f
1795(i)X
1 f
1820(,)X
2 f
1856(f)X
9 f
1895(\242)X
1 f
1917(\()X
2 f
1946(i)X
1 f
1978(\)\).)X
2107(It)X
2188(is)X
2274(not)X
2414(surprising)X
2793(that)X
2953(the)X
3088(class)X
3285(of)X
3384(functions)X
3738(comput-)X
576 4424(able)N
753(by)X
871(a)X
940(Turing)X
1210(machine)X
1540(running)X
1845(in)X
1945(time)X
2 f
2134(t)X
1 f
2173(\()X
2 f
2202(n)X
1 f
2253(\),)X
2335(for)X
2468(any)X
2626(computable)X
3069(function)X
2 f
3394(t)X
1 f
3419(,)X
3472(can)X
3625(be)X
3739(extrapo-)X
576 4584(lated)N
770(\(by)X
909(identi\256cation)X
1398(by)X
1508(enumeration\).)X
3 f
576 4904(4.)N
686(Rissanen's)X
1108(Minimum)X
1507(Description)X
1964(Length)X
2255(Principle)X
1 f
776 5092(Solomonoff's)N
1284(ideas)X
1489(about)X
1709(inductive)X
2062(reasoning)X
2428(have)X
2618(explicitly)X
2977(served)X
3231(as)X
3329(guiding)X
3624(principle)X
3963(in)X
576 5252(Rissanen's)N
1013(development)X
1525(of)X
1653(Minimum)X
2062(description)X
2509(length)X
2785(\(MDL\))X
3093(principle.)X
3506(Let)X
3679(us)X
3812(derive)X
576 5412(Rissanen's)N
985(MDL)X
1208(principle)X
1550(from)X
1749(Solomonoff's)X
2261(induction)X
2624(principle.)X
3011(For)X
3162(simplicit)X
3473(y,)X
3568(we)X
3699(deal)X
3875(with)X
576 5572(only)N
758(non-adaptive)X
1243(models.)X
1545(A)X
1633(non-adaptive)X
2118(model)X
2363(is)X
2446(a)X
2509(model)X
2 f
2754(P)X
1 f
2822(\()X
2 f
2851(D)X
1 f
9 f
2935(|)X
2967(q)X
1 f
3013(\))X
3066(where)X
3304(the)X
3436(parameter)X
3812(vector)X
9 f
576 5732(q)N
2 f
9 f
622(=)X
1 f
9 f
670(q)X
1 f
716(\()X
2 f
745(D)X
1 f
815(\))X
866(is)X
947(estimated)X
1308(from)X
2 f
1501(n)X
1 f
1567(observed)X
1906(data)X
2075(points)X
2313(denoted)X
2614(by)X
2 f
2724(D)X
1 f
2787(.)X
ep
%%Page: 30 30
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(30)X
2370(-)X
776 736(Scientists)N
1145(formulate)X
1518(their)X
1710(theories)X
2019(in)X
2118(two)X
2280(steps:)X
2511(\256rstly)X
2747(a)X
2817(scientist)X
3138(must,)X
3363(based)X
3594(on)X
3713(scienti\256c)X
576 896(observations)N
1046(or)X
1144(given)X
1365(data,)X
1559(formulate)X
1927(alternative)X
2325(hypotheses,)X
2762(and)X
2913(secondly)X
3250(he)X
3357(selects)X
3616(one)X
3767(de\256nite)X
576 1056(hypothesis.)N
1025(This)X
1210(is)X
1297(the)X
1433(subject)X
1711(of)X
1812(inference)X
2167(in)X
2264(statistics.)X
2641(Statistici)X
2952(ans)X
3097(have)X
3291(developed)X
3681(many)X
3905(dif-)X
576 1216(ferent)N
818(principles)X
1203(to)X
1309(do)X
1433(this,)X
1619(like)X
1788(Occam's)X
2136(razor)X
2352(principle,)X
2724(the)X
2868(Maximum)X
3272(Likelihood)X
3696(principle,)X
576 1376(various)N
868(ways)X
1081(of)X
1187(using)X
1411(Bayesian)X
1767(formula)X
2079(with)X
2270(different)X
2607(prior)X
2812(distributions.)X
3332(No)X
3473(single)X
3718(principle)X
576 1536(turned)N
831(out)X
974(to)X
1073(be)X
1186(satisfactory)X
1622(in)X
1721(all)X
1840(situations.)X
2253(Philosophically)X
2833(speaking,)X
3197(Solomonoff's)X
3710(approach)X
576 1696(presents)N
895(an)X
1009(ideal)X
1212(way)X
1389(of)X
1493(solving)X
1784(induction)X
2149(problems)X
2509(using)X
2732(Bayes')X
3008(rule)X
3177(with)X
3366(the)X
3506(universal)X
3861(prior)X
576 1856(distribution.)N
1051(However,)X
1417(due)X
1566(to)X
1657(the)X
1787(non-computability)X
2462(of)X
2557(the)X
2687(universal)X
3032(prior)X
3225(function,)X
3563(such)X
3746(a)X
3807(theory)X
576 2016(cannot)N
834(be)X
940(directly)X
1233(used)X
1417(in)X
1509(practice.)X
1855(Some)X
2079(approximation)X
2617(is)X
2700(needed)X
2973(in)X
3066(the)X
3198(real)X
3354(world)X
3583(applications.)X
576 2176(Further,)N
881(from)X
1076(theory)X
1325(to)X
1418(inductive)X
1771(inference)X
2122(and)X
2273(statistical)X
2632(practice,)X
2957(there)X
3157(is)X
3240(still)X
3398(a)X
3460(big)X
3596(distance,)X
3930(for)X
576 2336(example,)N
919(concrete)X
1239(formulae)X
1579(are)X
1708(needed.)X
776 2524(Gold's)N
1046(principle)X
1391(was)X
1558(a)X
1628(particularly)X
2066(simple)X
2333(approximation)X
2880(to)X
2981(Solomonoff's)X
3497(induction)X
3863(-)X
3924(the)X
576 2684(sophisticated)N
1067(notion)X
1323(of)X
1426(probability)X
1844(distribution)X
2282(is)X
2371(replaced)X
2699(by)X
2817(linear)X
3048(enumeration.)X
3563(Now)X
3763(we)X
3894(will)X
576 2844(closely)N
859(follow)X
1122(Solomonoff's)X
1639(ideas,)X
1875(but)X
2021(substitute)X
2394(a)X
2467(`good')X
2735(computable)X
3181(approximation)X
3730(to)X
3 f
3833(m)X
1 f
3906(\()X
2 f
3935(x)X
1 f
3981(\).)X
576 3004(This)N
769(results)X
1035(in)X
1140(Rissanen's)X
3 f
1557(Minimum)X
1970(Description)X
2440(Length)X
2744(Principle)X
1 f
3123(\(MDL)X
3382(principle\).)X
3782(He)X
3919(not)X
576 3164(only)N
758(gives)X
969(the)X
1102(principle,)X
1463(more)X
1669(importantly)X
2107(he)X
2216(also)X
2384(gives)X
2596(the)X
2730(detailed)X
3036(formulas)X
3375(on)X
3489(how)X
3666(to)X
3761(use)X
3904(this)X
576 3324(principle.)N
957(This)X
1137(makes)X
1385(it)X
1458(possible)X
1770(to)X
1862(use)X
2002(the)X
2133(MDL)X
2351(principle)X
2688(in)X
2780(real)X
2935(problems.)X
3308(The)X
3468(principle)X
3805(can)X
3949(be)X
576 3484(intuitivel)N
897(y)X
963(stated)X
1191(as)X
1286(follows:)X
3 f
776 3672(Minimum)N
1183(Description)X
1648(Length)X
1947(Principle.)X
2 f
2365(The)X
2527(best)X
2699(theory)X
2954(to)X
3053(explain)X
3343(a)X
3417(set)X
3546(of)X
3646(data)X
3834(is)X
3924(the)X
576 3832(one)N
725(which)X
958(minimizes)X
1332(the)X
1462(sum)X
1625(of)X
1 f
576 4020(\(1\))N
2 f
776(the)X
906(length,)X
1171(in)X
1262(bits,)X
1434(of)X
1525(the)X
1655(description)X
2074(of)X
2165(the)X
2295(theory;)X
1 f
576 4208(\(2\))N
2 f
776(the)X
906(length,)X
1171(in)X
1262(bits,)X
1434(of)X
1525(data)X
1704(when)X
1912(encoded)X
2227(with)X
2402(the)X
2532(help)X
2706(of)X
2797(the)X
2927(theory.)X
1 f
776 4396(We)N
931(now)X
1115(develop)X
1427(this)X
1588(MDL)X
1816(principle)X
2163(from)X
2367(Bayes')X
2644(rule,)X
2836(Formula)X
3168(\(1\),)X
3325(using)X
3549(the)X
3690(Universal)X
576 4556(distribution)N
3 f
1006(m)X
1 f
1079(\()X
2 f
1108(x)X
1 f
1154(\).)X
1227(Recall)X
1475(Bayes')X
1741(formula:)X
2 f
1787 4831(P)N
1 f
1855(\()X
2 f
1884(H)X
1 f
9 f
1975(|)X
2 f
2021(D)X
1 f
2091(\))X
2 f
9 f
2142(=)X
1 f
2 f
2407 4892(P)N
1 f
2475(\()X
2 f
2504(D)X
1 f
2574(\))X
2 f
2234 4779(P)N
1 f
2302(\()X
2 f
2331(D)X
1 f
9 f
2422(|)X
2 f
2468(H)X
1 f
2538(\))X
2 f
2581(P)X
1 f
2649(\()X
2 f
2678(H)X
1 f
2748(\))X
10 f
2220 4805(hhhhhhhhhhhhh)N
2 f
2798 4831(.)N
1 f
576 5115(Here)N
2 f
776(H)X
1 f
869(is)X
958(an)X
1071(hypothesis,)X
1501(here)X
1683(a)X
1753(probability)X
2172(distribution,)X
2633(which)X
2879(we)X
3012(assume)X
3302(to)X
3402(be)X
3516(computable)X
3959(or)X
576 5275(anyway)N
878(semicomputable,)X
1508(and)X
2 f
1664(D)X
1 f
1756(the)X
1892(observed)X
2237(data.)X
2456(We)X
2606(must)X
2806(choose)X
3078(the)X
3214(hypothesis)X
2 f
3619(H)X
1 f
3710(such)X
3899(that)X
2 f
576 5435(P)N
1 f
644(\()X
2 f
673(H)X
1 f
9 f
764(|)X
2 f
810(D)X
1 f
880(\))X
931(is)X
1012(maximized.)X
1471(First)X
1655(we)X
1779(take)X
1948(the)X
2078(negative)X
2399(logarithm)X
2765(on)X
2875(both)X
3054(sides)X
3252(of)X
3347(the)X
3477(formula:)X
2 f
9 f
1432 5675(-)N
1 f
1480(log)X
2 f
1593(P)X
1 f
1661(\()X
2 f
1690(H)X
1 f
9 f
1781(|)X
2 f
1813(D)X
1 f
1883(\))X
2 f
9 f
1912(=-)X
1 f
2008(log)X
2 f
2121(P)X
1 f
2189(\()X
2 f
2218(D)X
1 f
9 f
2309(|)X
2 f
2341(H)X
1 f
2411(\))X
2 f
9 f
2440(-)X
1 f
2488(log)X
2 f
2601(P)X
1 f
2669(\()X
2 f
2698(H)X
1 f
2768(\))X
2 f
9 f
2797(+)X
1 f
2845(log)X
2 f
2958(P)X
1 f
3026(\()X
2 f
3055(D)X
1 f
3125(\).)X
576 5915(Since)N
2 f
800(P)X
1 f
868(\()X
2 f
897(D)X
1 f
967(\))X
1024(can)X
1174(be)X
1285(considered)X
1694(as)X
1795(a)X
1862(normalizing)X
2317(factor,)X
2572(we)X
2703(ignore)X
2957(it)X
3036(in)X
3134(the)X
3271(following)X
3643(discussion.)X
ep
%%Page: 31 31
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(31)X
2370(-)X
576 736(Since)N
796(we)X
922(are)X
1053(only)X
1234(concerned)X
1619(with)X
1800(maximizing)X
2247(the)X
2379(term)X
2 f
2565(P)X
1 f
2633(\()X
2 f
2662(H)X
1 f
9 f
2753(|)X
2 f
2785(D)X
1 f
2855(\))X
2908(or,)X
3027(equivalently,)X
2 f
3509(minimizing)X
1 f
3924(the)X
576 896(term)N
2 f
9 f
760(-)X
1 f
808(log)X
2 f
921(P)X
1 f
989(\()X
2 f
1018(H)X
1 f
9 f
1109(|)X
2 f
1141(D)X
1 f
1211(\),)X
1284(this)X
1434(is)X
1515(equivalent)X
1905(to)X
1996(minimizi)X
2317(ng)X
2 f
9 f
1839 1136(-)N
1 f
1901(log)X
2 f
2028(P)X
1 f
2096(\()X
2 f
2125(D)X
1 f
9 f
2216(|)X
2 f
2248(H)X
1 f
2318(\))X
2 f
9 f
2361(-)X
1 f
2423(log)X
2 f
2550(P)X
1 f
2618(\()X
2 f
2647(H)X
1 f
2717(\).)X
576 1376(Now)N
779(to)X
881(get)X
1022(the)X
1163(minimum)X
1541(description)X
1966(length)X
2220(principle,)X
2590(we)X
2726(only)X
2917(need)X
3117(to)X
3220(explain)X
3514(the)X
3656(above)X
3900(two)X
576 1536(terms)N
817(in)X
931(the)X
1084(sum)X
1275(properly.)X
1639(According)X
2054(to)X
2167(Solomonoff,)X
2654(when)X
2 f
2888(P)X
1 f
2986(is)X
3089(semicomputable,)X
3734(then)X
3930(we)X
576 1696(approximate)N
2 f
1042(P)X
1 f
1121(by)X
3 f
1235(m)X
1 f
1308(.)X
1378(The)X
1541(prior)X
1738(probability)X
2 f
2152(P)X
1 f
2220(\()X
2 f
2249(H)X
1 f
2319(\))X
2374(is)X
2459(set)X
2583(to)X
3 f
2678(m)X
1 f
2751(\()X
2 f
2780(H)X
1 f
2850(\))X
2 f
9 f
2879(=)X
1 f
2927(2)X
2 f
8 s
9 f
1661(-)Y
2 f
3006(K)X
1 f
3059(\()X
2 f
3080(H)X
1 f
3131(\))X
2 f
9 f
3152(\261)X
2 f
3187(O)X
1 f
3243(\(1\))X
11 s
3324 1696(,)N
3372(where)X
2 f
3612(K)X
1 f
3685(\()X
2 f
3714(H)X
1 f
3784(\))X
3839(is)X
3924(the)X
576 1856(pre\256x)N
814(Kolmogorov)X
1297(complexity)X
1728(of)X
2 f
1834(H)X
1 f
1897(.)X
1974(That)X
2169(is,)X
2 f
9 f
2283(-)X
1 f
2331(log)X
2 f
2444(P)X
1 f
2512(\()X
2 f
2541(H)X
1 f
2611(\))X
2673(is)X
2765(precisely)X
3116(the)X
2 f
3257(length)X
1 f
3511(of)X
3616(a)X
3687(minimum)X
2 f
576 2016(pre\256x)N
798(code)X
1 f
964(,)X
1008(or)X
1103(program,)X
1445(of)X
1540(the)X
1670(hypothesis)X
2 f
2069(H)X
1 f
2132(.)X
776 2204(Similar)N
1063(argument)X
1422(applies)X
1698(to)X
1793(term)X
2 f
9 f
1981(-)X
1 f
2029(log)X
2 f
2142(P)X
1 f
2210(\()X
2 f
2239(D)X
1 f
9 f
2330(|)X
2 f
2362(H)X
1 f
2432(\).)X
2531(Assuming)X
2 f
2914(P)X
1 f
2994(is)X
3079(semicomputable,)X
3706(using)X
3924(the)X
576 2364(conditional)N
1001(version)X
1286(of)X
1385(\(3\))X
1513(and)X
1666(\(4\),)X
1816(we)X
1944(know)X
2165(that)X
2324(the)X
2458(universal)X
2807(semimeasure)X
3 f
3293(m)X
1 f
3366(\()X
2 f
3395(x)X
1 f
3441(\))X
3496(has)X
3639(the)X
3773(follow-)X
576 2524(ing)N
711(properties.)X
576 2712(\(a\))N
776(There)X
1003(is)X
1084(a)X
1145(constant)X
2 f
1461(c)X
1 f
1500(,)X
1544(such)X
1727(that)X
3 f
1882(m)X
1 f
1955(\()X
2 f
1984(D)X
1 f
9 f
2075(|)X
2 f
2121(H)X
1 f
2191(\))X
9 f
2234(\263)X
2 f
2304(cP)X
1 f
2411(\()X
2 f
2440(D)X
1 f
9 f
2531(|)X
2 f
2577(H)X
1 f
2647(\).)X
576 2900(\(b\))N
776(The)X
2 f
935(P)X
1 f
989(-probability)X
1428(that)X
3 f
1583(m)X
1 f
1656(\()X
2 f
1685(D)X
1 f
9 f
1776(|)X
2 f
1822(H)X
1 f
1892(\))X
9 f
1935(\243)X
2 f
1997(kP)X
1 f
2104(\()X
2 f
2133(D)X
1 f
9 f
2224(|)X
2 f
2270(H)X
1 f
2340(\))X
2391(is)X
2472(at)X
2558(least)X
2742(1)X
2 f
9 f
2800(-)X
1 f
2862(1)X
2 f
2913(/)X
2945(k)X
1 f
2984(.)X
576 3088(By)N
702(a)X
764(conditional)X
1185(version)X
1467(of)X
1564(Theorem)X
1906(3,)X
3 f
1996(m)X
1 f
2069(\()X
2 f
2098(D)X
1 f
9 f
2189(|)X
2 f
2235(H)X
1 f
2305(\))X
2 f
9 f
2348(=)X
1 f
2410(2)X
2 f
8 s
9 f
3053(-)Y
2 f
2489(K)X
1 f
2542(\()X
2 f
2563(D)X
1 f
9 f
2629(|)X
2 f
2652(H)X
1 f
2703(\))X
2 f
9 f
2724(\261)X
2 f
2759(O)X
1 f
2815(\(1\))X
11 s
2896 3088(.)N
2942(Hence)X
3190(again)X
3405(2)X
2 f
8 s
9 f
3053(-)Y
2 f
3484(K)X
1 f
3537(\()X
2 f
3558(D)X
1 f
9 f
3624(|)X
2 f
3647(H)X
1 f
3698(\))X
11 s
3750 3088(is)N
3833(a)X
3896(rea-)X
576 3248(sonable)N
869(approximation)X
1408(of)X
2 f
1505(P)X
1 f
1573(\()X
2 f
1602(D)X
1 f
9 f
1693(|)X
2 f
1725(H)X
1 f
1795(\),)X
1870(and)X
2021(minimizi)X
2342(ng)X
2 f
9 f
2453(-)X
1 f
2501(log)X
2 f
2614(P)X
1 f
2682(\()X
2 f
2711(D)X
1 f
9 f
2802(|)X
2 f
2834(H)X
1 f
2904(\))X
2956(can)X
3101(be)X
3207(considered)X
3611(as)X
3707(minimiz-)X
576 3408(ing)N
2 f
716(K)X
1 f
789(\()X
2 f
818(D)X
1 f
9 f
909(|)X
2 f
941(H)X
1 f
1011(\),)X
2 f
1089(i.e.)X
1 f
1197(,)X
1246(\256nding)X
1523(an)X
2 f
1633(H)X
1 f
1723(such)X
1911(that)X
2071(the)X
2206(description)X
2626(length,)X
2897(or)X
2998(the)X
3134(Kolmogorov)X
3612(complexity,)X
576 3568(of)N
2 f
675(D)X
1 f
764(given)X
2 f
986(H)X
1 f
1075(is)X
1160(minimized.)X
1609(The)X
1771(term)X
2 f
9 f
1958(-)X
1 f
2006(log)X
2 f
2133(P)X
1 f
2194(\()X
2 f
2223(D)X
1 f
9 f
2314(|)X
2 f
2346(H)X
1 f
2416(\))X
2470(can)X
2617(also)X
2784(be)X
2892(thought)X
3187(as)X
3285(the)X
2 f
3418(ideal)X
3620(code)X
3811(length)X
1 f
576 3728(for)N
719(describing)X
1127(data)X
2 f
1315(D)X
1 f
1378(,)X
1441(given)X
1678(hypothesis)X
2 f
2096(H)X
1 f
2159(.)X
2222(Such)X
2439(pre\256x)X
2685(code)X
2892(length)X
3154(can)X
3317(be)X
3441(achieved)X
3795(by)X
3924(the)X
576 3888(Shannon-Fano)N
1114(code.)X
1349(The)X
1511(term)X
2 f
9 f
1720(-)X
1 f
1768(log)X
2 f
1881(P)X
1 f
1949(\()X
2 f
1978(D)X
1 f
9 f
2069(|)X
2 f
2101(H)X
1 f
2171(\),)X
2247(also)X
2414(known)X
2678(as)X
2775(the)X
2 f
2907(self-information)X
1 f
3476(,)X
3522(in)X
3615(information)X
576 4048(theory,)N
849(and)X
1002(the)X
1136(negative)X
1461(log)X
1600(likelihood)X
1985(in)X
2080(statistics,)X
2433(can)X
2581(now)X
2758(be)X
2867(regarded)X
3200(as)X
3299(the)X
3433(number)X
3728(of)X
3827(bits)X
3982(it)X
576 4208(takes)N
779(to)X
870(redescribe)X
1253(or)X
1348(encode)X
2 f
1619(D)X
1 f
1704(with)X
1883(an)X
1988(ideal)X
2182(code)X
2370(relative)X
2657(to)X
2 f
2748(H)X
1 f
2811(.)X
776 4396(In)N
872(the)X
1004(original)X
1303(Solomonoff)X
1748(approach,)X
2 f
2116(H)X
1 f
2203(in)X
2296(general)X
2579(is)X
2662(a)X
2725(Turing)X
2989(machine.)X
3334(In)X
3431(practice)X
3734(we)X
3860(must)X
576 4556(avoid)N
800(such)X
989(an)X
1100(overly)X
1353(general)X
1640(approach)X
1990(in)X
2087(order)X
2300(to)X
2397(keep)X
2591(things)X
2835(computable.)X
3318(In)X
3418(different)X
3748(applica-)X
576 4716(tions,)N
792(the)X
922(hypothesis)X
2 f
1321(H)X
1 f
1406(can)X
1550(mean)X
1763(many)X
1981(different)X
2306(things.)X
2566(For)X
2710(example,)X
3053(if)X
3129(we)X
3253(infer)X
3441(decision)X
3758(trees,)X
2 f
3969(H)X
1 f
576 4876(is)N
658(a)X
720(decision)X
1037(tree;)X
1217(In)X
1313(case)X
1487(of)X
1583(learning)X
1895(\256nite)X
2100(automata,)X
2 f
2469(H)X
1 f
2555(can)X
2699(be)X
2804(a)X
2865(\256nite)X
3069(automaton;)X
3511(In)X
3628(case)X
3801(we)X
3925(are)X
576 5036(interested)N
951(in)X
1052(learning)X
1373(Boolean)X
1699(formulae,)X
2071(then)X
2 f
2255(H)X
1 f
2350(may)X
2534(be)X
2649(a)X
2720(Boolean)X
3046(formula;)X
3382(If)X
3473(we)X
3608(are)X
3748(\256tting)X
3993(a)X
576 5196(polynomial)N
1006(to)X
1102(a)X
1168(set)X
1293(of)X
1393(points,)X
1658(then)X
2 f
1837(H)X
1 f
1927(may)X
2106(be)X
2216(a)X
2282(polynomial)X
2712(of)X
2812(some)X
3025(degree;)X
3311(In)X
3411(general)X
3697(statistical)X
576 5356(applications,)N
1055(one)X
1212(assumes)X
1535(that)X
2 f
1698(H)X
1 f
1791(is)X
1880(some)X
2096(model)X
2 f
2348(H)X
1 f
2425(\()X
9 f
2454(q)X
1 f
2500(\))X
2560(with)X
2748(a)X
2818(set)X
2947(of)X
3051(parameters)X
9 f
3468(q)X
2 f
9 f
3514(=)X
2 f
3562({)X
1 f
9 f
3597(q)X
8 s
1 f
3643 5373(1)N
11 s
3682 5356(,)N
3733 5330(.)N
3777(.)X
3821(.)X
3865 5356(,)N
9 f
3894(q)X
2 f
8 s
3940 5373(k)N
11 s
3975 5356(})N
1 f
4010(,)X
576 5516(where)N
813(the)X
944(number)X
2 f
1236(k)X
1 f
1298(may)X
1473(vary)X
1652(and)X
1802(in\257uence)X
2148(the)X
2279(\(descriptional\))X
2816(complexity)X
3237(of)X
2 f
3333(H)X
1 f
3410(\()X
9 f
3439(q)X
1 f
3485(\).)X
3581(In)X
3676(such)X
3859(case,)X
576 5676(from)N
2 f
9 f
1898 5916(-)N
1 f
1946(log)X
2 f
2059(P)X
1 f
2127(\()X
2 f
2156(D)X
1 f
9 f
2240(|)X
2272(q)X
1 f
2318(\))X
2 f
9 f
2347(-)X
1 f
2395(log)X
2 f
2508(P)X
1 f
2576(\()X
9 f
2605(q)X
1 f
2651(\),)X
ep
%%Page: 32 32
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(32)X
2370(-)X
576 740(using)N
797(Taylor)X
1062(expansion)X
1449(at)X
1544(the)X
1683(point)X
1896(of)X
2000(optimal)X
9 f
2302(q)X
12 s
1 f
2309 710(\303)N
11 s
2379 740(\(for)N
2541(best)X
2714(maximum)X
3104(likelihood)X
3494(estimator\),)X
3905(and)X
576 900(taking)N
824(only)X
1008(dominant)X
1369(terms,)X
1613(Rissanen)X
1957(has)X
2100(derived)X
2390(a)X
2455(formula)X
2760(for)X
2888(the)X
3022(minimum)X
3393(description)X
3811(length)X
576 1060(as)N
8 s
9 f
1816 1399(q)N
1 f
1849(,)X
2 f
1870(k)X
1 f
11 s
1788 1335(min)N
2 f
1926({)X
9 f
1961(-)X
1 f
2009(log)X
2 f
2122(P)X
1 f
2190(\()X
2 f
2219(D)X
1 f
9 f
2303(|)X
2335(q)X
1 f
2381(\))X
2 f
9 f
2410(+)X
1 f
2480 1396(2)N
2480 1283(1)N
10 f
2466 1309(h)N
2493(h)X
2 f
2545 1335(k)N
1 f
2591(log)X
2 f
2704(n)X
2755(})X
1 f
2790(,)X
576 1621(where)N
2 f
815(k)X
1 f
879(is)X
963(the)X
1096(number)X
1390(of)X
1488(parameters)X
1899(in)X
9 f
1994(q)X
2 f
9 f
2040(=)X
2 f
2088({)X
1 f
9 f
2123(q)X
8 s
1 f
2169 1638(1)N
11 s
2208 1621(,)N
2259 1595(.)N
2303(.)X
2347(.)X
2391 1621(,)N
9 f
2420(q)X
2 f
8 s
2466 1638(k)N
11 s
2501 1621(})N
1 f
2536(,)X
2584(and)X
2 f
2737(n)X
1 f
2807(is)X
2892(number)X
3187(of)X
3286(observations)X
3757(\(or)X
3885(data)X
576 1781(points\))N
847(contained)X
1216(in)X
2 f
1311(D)X
1 f
1374(.)X
1444(At)X
1558(the)X
1691(optimal)X
2 f
1987(k)X
1 f
2051(and)X
9 f
2203(q)X
1 f
2249(,)X
2296(the)X
2429(term)X
2616(1)X
2 f
2667(/)X
1 f
2699(2)X
2 f
2757(k)X
1 f
2817(log)X
2 f
2944(n)X
1 f
3013(is)X
3097(called)X
3333(the)X
3466(optimum)X
3811(model)X
576 1941(cost.)N
776 2129(Since)N
2 f
1000(K)X
1 f
1073(\()X
2 f
1102(H)X
1 f
1172(\))X
1229(is)X
1316(not)X
1457(computable)X
1898(and)X
2054(hard)X
2239(to)X
2337(approximate,)X
2829(Rissanen)X
3176(suggested)X
3552(the)X
3689(following)X
576 2289(approach.)N
977(First)X
1174(convert)X
1473(\(or)X
1609(encode\))X
2 f
1921(H)X
1 f
2018(to)X
2121(a)X
2194(positive)X
2508(integer)X
2787(in)X
2 f
2890(N)X
9 f
2970(=)X
2 f
3032({)X
1 f
3067(1,)X
3140(2,)X
3235 2263(.)N
3279(.)X
3323(.)X
2 f
3367 2289(})N
1 f
3402(.)X
3480(Then)X
3695(we)X
3831(try)X
3963(to)X
576 2449(assign)N
821(prior)X
1017(distribution)X
1450(to)X
1544(each)X
1731(integer)X
2002(in)X
2 f
2097(N)X
1 f
2156(.)X
2226(Jeffreys)X
2529([)X
2584(Jeffreys)X
2887(Theory)X
3167(of)X
3266(Probability)X
3685(suggested)X
576 2609(to)N
676(assign)X
927(probability)X
1346(1)X
2 f
1397(/)X
1429(n)X
1 f
1504(to)X
1604(integer)X
2 f
1880(n)X
1 f
(.)S
1999(But)X
2158(this)X
2317(results)X
2578(an)X
2692(improper)X
3045(distribution)X
3483(since)X
3694(the)X
3832(series)X
16 s
9 f
576 2795(S)N
11 s
1 f
666 2769(1)N
2 f
717(/)X
749(n)X
1 f
818(diverges.)X
1185(We)X
1332(modify)X
1612(Jeffreys)X
1914(distribution.)X
2369(It)X
2448(is)X
2532(possible,)X
2868(by)X
2982(iterating)X
3303(the)X
3437(idea)X
3610(of)X
3709(encoding)X
2 f
576 2941(n)N
1 f
647(\(viewed)X
957(as)X
1057(the)X
1192(corresponding)X
2 f
1722(n)X
1 f
(th)S
1862(binary)X
2114(string\))X
2371(as)X
2 f
2471(n)X
1 f
9 f
2522(\242)X
2 f
9 f
2558(=)X
2 f
2620(l)X
1 f
2659(\()X
2 f
2688(n)X
1 f
2739(\))X
10 f
2628 2860(h)N
2636(hhh)X
2 f
2782 2941(n)N
1 f
(,)S
2875(to)X
2971(obtain)X
3219(a)X
3285(pre\256x-code)X
3712(such)X
3899(that)X
2 f
576 3101(L)N
1 f
639(\()X
2 f
668(n)X
1 f
719(\))X
770(denotes)X
1061(the)X
1191(length)X
1434(of)X
1529(the)X
1659(code)X
1847(for)X
2 f
1971(n)X
1 f
(,)S
2059(with)X
2 f
2238(L)X
1 f
2301(\()X
2 f
2330(n)X
1 f
2381(\))X
2432(de\256ned)X
2713(by)X
2 f
1767 3341(l*)N
1 f
1850(\()X
2 f
1879(n)X
1 f
1930(\))X
2 f
9 f
1981(=)X
1 f
2051(log)X
2 f
2178(n)X
9 f
2243(+)X
1 f
2305(loglog)X
2 f
2545(n)X
9 f
2610(+)X
1 f
2680 3315(.)N
2724(.)X
2768(.)X
2812 3341(,)N
576 3581(all)N
687(positive)X
989(terms,)X
1229(and)X
2 f
1944 3821(L)N
1 f
2007(\()X
2 f
2036(n)X
1 f
2087(\))X
2 f
9 f
2138(=)X
1 f
2 f
2208(l*)X
1 f
2284(\()X
2 f
2313(n)X
1 f
2364(\))X
2 f
9 f
2407(+)X
1 f
2469(log)X
2 f
2596(c)X
1 f
2635(,)X
576 4061(where)N
2 f
812(c)X
9 f
865(=)X
1 f
913(2.865064)X
1265 4035(.)N
1309(.)X
1353(.)X
1397 4061(.)N
1441(Viz,)X
1612(it)X
1684(can)X
1828(be)X
1933(proved)X
2199([)X
2250(Rissanen)X
2590(Universal)X
2954(prior)X
3147(integers)X
3448(that:)X
2 f
8 s
2061 4433(n)N
9 f
2103(=)X
1 f
2138(1)X
16 s
9 f
2077 4369(S)N
8 s
2092 4260(\245)N
11 s
1 f
2169 4343(2)N
2 f
8 s
9 f
4308(-)Y
2 f
2248(l*)X
1 f
2308(\()X
2 f
2329(n)X
1 f
2366(\))X
11 s
2 f
9 f
2416 4343(=)N
1 f
2 f
2486(c.)X
1 f
576 4655(Therefore,)N
966(the)X
1096(existence)X
1446(of)X
1541(a)X
1602(pre\256x-code)X
2024(as)X
2119(claimed)X
2421(follows)X
2707(from)X
2900(Kraft's)X
3170(Inequality.)X
776 4843(Assign)N
1051(prior)X
1253(probability)X
2 f
1672(P)X
1 f
1740(\()X
2 f
1769(n)X
1 f
1820(\))X
2 f
9 f
1863(=)X
1 f
1925(2)X
2 f
8 s
9 f
4808(-)Y
2 f
2004(L)X
1 f
2050(\()X
2 f
2071(n)X
1 f
2108(\))X
11 s
2167 4843(to)N
2267(each)X
2459(integer)X
2 f
2736(n)X
1 f
(.)S
2856(We)X
3010(obtain)X
3263(the)X
3403(following)X
3778(desired)X
576 5021(properties:)N
980(\(a\))X
16 s
9 f
1104 5047(S)N
2 f
8 s
1185 5065(n)N
9 f
1227(=)X
1 f
1262(1)X
9 f
1185 4988(\245)N
11 s
1 f
1301 5021(2)N
2 f
8 s
9 f
4986(-)Y
2 f
1380(L)X
1 f
1426(\()X
2 f
1447(n)X
1 f
1484(\))X
2 f
11 s
9 f
1526 5021(=)N
1 f
1588(1;)X
1684(and)X
1838(\(b\))X
1966(integers)X
2 f
2271(n)X
1 f
2341(are)X
2474(coded)X
2710(by)X
2824(a)X
2889(pre\256x)X
3120(code.)X
3334(Hence,)X
3606(descriptions)X
576 5181(of)N
673(two)X
829(integers,)X
2 f
1154(n)X
1 f
8 s
1208 5198(1)N
11 s
1271 5181(and)N
2 f
1422(n)X
1 f
8 s
1476 5198(2)N
11 s
1515 5181(,)N
1561(can)X
1707(be)X
1814(just)X
1966(concatenated)X
2450(to)X
2543(produce)X
2850(the)X
2982(code)X
3172(for)X
3298(the)X
3431(pair)X
3593(\()X
2 f
3622(n)X
1 f
8 s
3676 5198(1)N
11 s
3715 5181(,)N
2 f
3744(n)X
1 f
8 s
3798 5198(2\))N
11 s
3858 5181(,)N
3905(and)X
576 5341(so)N
676(on.)X
830(The)X
989(decoding)X
1334(process)X
1619(is)X
1700(trivial.)X
776 5529(Using)N
1011(the)X
1145(MDL)X
1366(principle,)X
1728(Wax)X
1920(and)X
2073(Rissanen)X
2417(\(according)X
2819(to)X
2914(Wax\))X
3135(and)X
3288(Quinlan)X
3598(and)X
3751(Rivest)X
4003([)X
576 5689(Quinlan)N
894(Rivest)X
1154(have)X
1354(developed)X
1749(procedures)X
2167(to)X
2269(infer)X
2468(decision)X
2795(trees.)X
3038(Other)X
3271(work)X
3484(by)X
3605(Wax)X
3804([)X
3866(Wax)X
576 5849(Detection)N
941(coherent)X
1266(and)X
1415(by)X
1525(Gao)X
1693(and)X
1842(Li)X
1943([)X
1994(gao)X
2143(li)X
2215(applied)X
2497(MDL)X
2714(principle)X
3050(to)X
3141(recognition)X
3565(problems.)X
ep
%%Page: 33 33
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(33)X
2370(-)X
3 f
776 736(Example.)N
1 f
1157(We)X
1305(sketch)X
1556(an)X
1665(initial)X
1899(experiment)X
2322(we)X
2450([)X
2505(gao)X
2658(li)X
2734(have)X
2926(performed)X
3318(in)X
3413(on-line)X
3690(handwrit-)X
576 896(ten)N
711(character)X
1060(learning)X
1376(using)X
1594(the)X
1729(MDL)X
1950(principle.)X
2334(Inputing)X
2659(Chinese)X
2969(characters)X
3351(into)X
3515(computers)X
3908(is)X
3993(a)X
576 1056(dif\256cult)N
893(task.)X
1116(There)X
1358(are)X
1502(at)X
1603(least)X
1803(5,000)X
2039(characters)X
2433(in)X
2540(daily)X
2755(use,)X
2932(all)X
3059(of)X
3170(different)X
3511(shapes.)X
3827(Many)X
576 1216(methods)N
900(have)X
1091(been)X
1282(invented)X
1611(for)X
1738(key-board)X
2119(input.)X
2348(Some)X
2574(have)X
2765(been)X
2956(successful)X
3341(in)X
3434(the)X
3566(limited)X
3842(sense)X
576 1376(that)N
738(they)X
919(can)X
1070(be)X
1182(used)X
1372(by)X
1489(trained)X
1763(typists)X
2023(only.)X
2253(Handwriting)X
2728(input)X
2940(is)X
3029(an)X
3142(alternative)X
3545(choice.)X
3827(Many)X
576 1536(such)N
759(systems)X
1060(have)X
1248(been)X
1436(built)X
1621(with)X
1800(various)X
2081(recognition)X
2505(rates.)X
776 1724(We)N
922([)X
975(gao)X
1126(li)X
1200(have)X
1390(implemented)X
1876(such)X
2062(a)X
2126(system)X
2396(that)X
2554(learns)X
2789(handwritten)X
3235(characters)X
3616(from)X
3812(exam-)X
576 1884(ples)N
742(under)X
966(the)X
1098(guidance)X
1440(of)X
1537(the)X
1669(MDL)X
1888(principle.)X
2269(We)X
2414(now)X
2588(sketch)X
2836(a)X
2898(simple)X
3157(experiment)X
3577(we)X
3702(have)X
3891(per-)X
576 2044(formed.)N
904(An)X
1041(input)X
1253(character)X
1605(is)X
1694(drawn)X
1943(on)X
2061(a)X
2130(digitizer)X
2455(board)X
2686(with)X
2874(200/inch)X
3214(resolution)X
3598(in)X
3698(both)X
3886(hor-)X
576 2204(izontal)N
851(and)X
1012(vertical)X
1310(directions.)X
1735(The)X
1905(system)X
2183(learns)X
2426(a)X
2498(character)X
2853(from)X
3057(examples.)X
3467(The)X
3637(basic)X
3851(algo-)X
576 2364(rithm)N
792(involves)X
1115(low)X
1271(level)X
1467(preprocessing,)X
2001(scaling,)X
2297(forming)X
2605(a)X
2668(prototype)X
3030(of)X
3127(a)X
3190(character)X
3536(\(for)X
3692(learning\),)X
576 2524(elastic)N
827(matching)X
1181(\(for)X
1337(recognizing\),)X
1829(and)X
1981(so)X
2084(on.)X
2241(At)X
2354(the)X
2487(stage)X
2693(of)X
2791(forming)X
3100(a)X
3164(prototype)X
3527(of)X
3625(a)X
3688(character,)X
576 2684(we)N
704(have)X
896(to)X
991(decide)X
1248(on)X
1363(the)X
2 f
1498(feature)X
1775(extraction)X
2160(intervals)X
1 f
2469(.)X
2518(Then)X
2726(we)X
2855(code)X
3048(a)X
3114(character)X
3463(into)X
3628(a)X
3694(prototype)X
576 2844(so)N
676(that)X
831(future)X
1063(inputs)X
1301(are)X
1430(classi\256ed)X
1780(according)X
2149(to)X
2240(their)X
2424(\(elastic)X
2701(Hamming\))X
3105(distance)X
3416(to)X
3507(the)X
3637(prototypes.)X
776 3032(Handwritten)N
1243(characters)X
1626(are)X
1760(usually)X
2042(quite)X
2246(arbitrary)X
2576(and)X
2730(prone)X
2957(to)X
3053(lots)X
3208(of)X
3308(noise.)X
3566(If)X
3652(the)X
3788(feature)X
576 3192(extraction)N
964(interval)X
1269(is)X
1363(very)X
1554(small,)X
1803(then)X
1990(the)X
2133(algorithm)X
2511(will)X
2683(be)X
2800(very)X
2990(sensitive)X
3333(to)X
3436(errors)X
3674(and)X
3835(slight)X
576 3352(changes)N
882(in)X
974(the)X
1105(recognition)X
1530(phase,)X
1775(causing)X
2067(low)X
2222(recognition)X
2647(rate.)X
2824(If)X
2905(the)X
3036(feature)X
3303(extraction)X
3679(interval)X
3973(is)X
576 3512(very)N
766(large,)X
998(then)X
1184(it)X
1268(becomes)X
1609(less)X
1774(likely)X
2009(that)X
2175(we)X
2310(extract)X
2583(the)X
2724(essential)X
3061(features)X
3372(of)X
3478(a)X
3550(character)X
3905(and)X
576 3672(hence)N
815(we)X
951(get)X
1093(a)X
1166(low)X
1333(recognition)X
1770(rate)X
1937(again.)X
2207(We)X
2364(must)X
2571(compromise.)X
3064(The)X
3236(compromise)X
3707(is)X
3801(on)X
3924(the)X
576 3832(basis)N
774(of)X
869(minimum)X
1236(description)X
1650(length)X
1893(of)X
1988(prototypes.)X
776 4020(We)N
931(proceeded)X
1325(as)X
1431(follows)X
1728(to)X
1830(establish)X
2172(an)X
2288(optimal)X
2592(feature)X
2869(selection)X
3216(interval.)X
3564(A)X
3661(set)X
3793(of)X
3900(186)X
576 4180(characters)N
956(drawings)X
1302(by)X
1414(one)X
1564(subject,)X
1859(exactly)X
2137(3)X
2204(examples)X
2560(for)X
2685(each)X
2869(of)X
2965(the)X
3096(62)X
3207(alphanumerical)X
3774(charac-)X
576 4340(ters,)N
748(were)X
941(recorded.)X
1315(The)X
1476(character)X
1822(drawings)X
2168(were)X
2362(stored)X
2601(in)X
2694(a)X
2757(standardized)X
3226(integer)X
3495(coordinate)X
3891(sys-)X
576 4500(tem)N
735(ranged)X
1000(from)X
1196(0)X
1265(to)X
1359(30)X
1472(in)X
1566(both)X
1748(x)X
1817(and)X
1969(y)X
2038(directions.)X
2455(These)X
2690(character)X
3037(drawings)X
3384(were)X
3579(then)X
3756(input)X
3963(to)X
576 4660(the)N
706(system)X
973(to)X
1064(establish)X
1395(a)X
1456(knowledge)X
1864(base,)X
2064(which)X
2302(formed)X
2579(the)X
2710(collection)X
3082(of)X
3178(prototypes)X
3573(with)X
3753(normal-)X
576 4820(ized)N
747(real)X
902(coordinates,)X
1353(based)X
1576(on)X
1687(some)X
1896(selected)X
2203(feature)X
2470(extraction)X
2846(interval.)X
3161(After)X
3369(the)X
3500(construction)X
3959(of)X
576 4980(knowledge)N
984(base)X
1162(was)X
1320(\256nished,)X
1643(the)X
1773(system)X
2040(was)X
2198(tested)X
2426(by)X
2536(having)X
2798(it)X
2870(classify)X
3161(the)X
3291(same)X
3494(set)X
3614(of)X
3710(character)X
576 5140(drawings.)N
950(If)X
1038(a)X
1107(character)X
1459(is)X
1548(misclassi\256ed,)X
2056(it)X
2136(is)X
2225(encoded)X
2548(using)X
2769(extra)X
2975(bits)X
3132(\(i.e.,)X
3320(the)X
3457(term)X
2 f
3648(P)X
1 f
3716(\()X
2 f
3745(D)X
1 f
9 f
3836(|)X
2 f
3882(H)X
1 f
3952(\)\).)X
576 5300(The)N
2 f
744(error)X
960(code)X
1157(length)X
1 f
1409(is)X
1499(the)X
1638(sum)X
1816(of)X
1920(the)X
2059(total)X
2248(number)X
2548(of)X
2652(points)X
2899(for)X
3032(all)X
3152(the)X
3291(incorrectly)X
3704(classi\256ed)X
576 5460(character)N
927(drawings.)X
1300(The)X
2 f
1466(model)X
1710(code)X
1 f
1905(length)X
2155(is)X
2243(the)X
2380(total)X
2567(number)X
2865(of)X
2967(points)X
3212(in)X
3310(all)X
3427(the)X
3563(prototypes)X
3963(in)X
576 5620(the)N
708(machine's)X
1094(knowledge)X
1504(base)X
1684(multipli)X
1966(ed)X
2074(by)X
2187(2.)X
2300(The)X
2462(factor)X
2692(of)X
2790(2)X
2859(comes)X
3109(from)X
3305(the)X
3438(fact)X
3595(that)X
3753(the)X
3886(pro-)X
576 5780(totype)N
830(coordinates)X
1269(are)X
1409(stored)X
1657(as)X
1763(real)X
1928(numbers.)X
2285(This)X
2474(takes)X
2687(twice)X
2910(as)X
3015(much)X
3243(memory)X
3569(\(in)X
3699(C\))X
3819(as)X
3924(the)X
ep
%%Page: 34 34
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(34)X
2370(-)X
576 736(character)N
938(drawing)X
1267(coordinates)X
1714(which)X
1970(are)X
2118(in)X
2228(integer)X
2514(form.)X
2748(The)X
2926(prototype)X
3305(coordinates)X
3752(are)X
3900(real)X
576 896(instead)N
854(of)X
955(integer)X
1228(numbers,)X
1581(to)X
1678(facilitate)X
2016(the)X
2152(elastic)X
2406(matching)X
2763(process)X
3054(to)X
3151(give)X
3331(small)X
3550(resolution)X
3930(for)X
576 1056(comparisons)N
1043(of)X
1138(classi\256cation.)X
776 1244(Thus,)N
996(both)X
1175(the)X
1305(model)X
1548(code)X
1736(length)X
1979(and)X
2128(the)X
2258(error)X
2450(code)X
2638(length)X
2881(are)X
3010(directly)X
3302(related)X
3565(to)X
3657(the)X
3788(feature)X
576 1404(extraction)N
952(interval.)X
1267(The)X
1426(smaller)X
1708(this)X
1858(interval,)X
2172(the)X
2302(more)X
2505(complex)X
2831(the)X
2961(prototypes,)X
3377(but)X
3512(the)X
3642(smaller)X
3924(the)X
576 1564(error)N
784(code)X
989(length.)X
1271(The)X
1447(effect)X
1686(is)X
1784(reversed)X
2120(if)X
2213(the)X
2360(feature)X
2643(extraction)X
3035(interval)X
3344(goes)X
3544(toward)X
3827(larger)X
576 1724(values.)N
849(Since)X
1071(the)X
2 f
1205(total)X
1394(code)X
1586(length)X
1 f
1833(is)X
1918(the)X
2052(sum)X
2225(of)X
2324(the)X
2458(two)X
2616(code)X
2808(lengths,)X
3110(there)X
3311(should)X
3571(be)X
3679(a)X
3743(value)X
3959(of)X
576 1884(the)N
709(feature)X
978(extraction)X
1356(interval)X
1651(gives)X
1862(a)X
1926(minimum)X
2297(for)X
2425(the)X
2559(total)X
2743(code)X
2935(length.)X
3226(This)X
3409(feature)X
3679(extraction)X
576 2044(interval)N
873(is)X
959(considered)X
1367(to)X
1463(be)X
1573(the)X
1708(`best')X
1935(one)X
2089(in)X
2185(the)X
2320(spirit)X
2528(of)X
2627(the)X
2761(MDL)X
2982(principle.)X
3366(The)X
3529(corresponding)X
576 2204(model,)N
844(or)X
942(knowledge)X
1353(base,)X
1556(is)X
1640(considered)X
2046(to)X
2140(be)X
2248(optimal)X
2544(in)X
2638(the)X
2771(sense)X
2986(that)X
3144(it)X
3220(contains)X
3540(enough)X
3825(of)X
3924(the)X
576 2364(essence)N
878(of)X
985(the)X
1127(raw)X
1292(data)X
1473(but)X
1620(eliminates)X
2017(most)X
2222(redundancy)X
2665(of)X
2771(the)X
2912(noise)X
3131(component)X
3556(from)X
3760(the)X
3901(raw)X
576 2524(data.)N
771(This)X
954(optimal)X
1251(feature)X
1521(extraction)X
1900(interval)X
2196(can)X
2344(be)X
2453(found)X
2684(empirically)X
3113(by)X
3227(carrying)X
3547(out)X
3687(the)X
3822(above)X
576 2684(described)N
940(build-and-test)X
1457(procedure)X
1834(repeatedly.)X
2249(That)X
2437(is,)X
2544(build)X
2752(the)X
2886(knowledge)X
3298(base,)X
3502(and)X
3655(then)X
3833(test)X
3982(it)X
576 2844(based)N
804(on)X
920(the)X
1056(same)X
1265(set)X
1391(of)X
1492(characters)X
1876(for)X
2006(which)X
2249(it)X
2327(was)X
2491(built.)X
2704(Repeat)X
2977(this)X
3133(for)X
3263(a)X
3330(number)X
3627(of)X
3729(different)X
576 3004(extraction)N
951(intervals.)X
776 3192(In)N
888(fact,)X
1081(this)X
1248(actual)X
1498(optimizati)X
1858(on)X
1985(process)X
2287(is)X
2385(implemented)X
2886(on)X
3014(the)X
3162(system)X
3447(and)X
3614(is)X
3713(available)X
576 3352(whenever)N
939(the)X
1069(user)X
1237(wants)X
1464(to)X
1555(call)X
1705(it.)X
1799(For)X
1943(our)X
2082(particular)X
2442(set)X
2562(of)X
2657(characters,)X
3057(the)X
3187(results)X
3 f
10 s
576 5552(Figure)N
823(3.)X
1 f
923(Optimization)X
11 s
576 5792(of)N
675(this)X
829(optimizati)X
1189(on)X
1303(are)X
1436(given)X
1658(in)X
1753(Figure)X
2009(3,)X
2101(which)X
2342(depicts)X
2618(three)X
2820(quantities:)X
3215(the)X
3349(model)X
3596(code)X
3789(length,)X
ep
%%Page: 35 35
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(35)X
2370(-)X
576 736(the)N
717(error)X
920(code)X
1119(length,)X
1395(and)X
1555(the)X
1695(total)X
1885(code)X
2083(length)X
2336(versus)X
2592(feature)X
2868(extraction)X
3253(interval)X
3555(\(SAMPLING)X
576 896(INTERVAL)N
1039(in)X
1132(the)X
1264(Figure\).)X
1569(For)X
1715(larger)X
1944(feature)X
2212(extraction)X
2589(intervals,)X
2940(the)X
3073(model)X
3319(code)X
3510(length)X
3756(is)X
3840(small)X
576 1056(but)N
713(most)X
909(of)X
1006(the)X
1138(character)X
1484(drawings)X
1830(are)X
1961(misclassi\256ed,)X
2463(giving)X
2713(a)X
2776(very)X
2956(large)X
3156(total)X
3338(code)X
3528(length.)X
3794(On)X
3924(the)X
576 1216(other)N
787(hand,)X
1010(when)X
1230(the)X
1368(feature)X
1642(extraction)X
2025(interval)X
2325(is)X
2414(at)X
2508(the)X
2646(small)X
2868(end)X
3026(of)X
3130(the)X
3269(scale,)X
3498(all)X
3618(the)X
3757(training)X
576 1376(characters)N
967(get)X
1110(correctly)X
1458(classi\256ed,)X
1843(and)X
2005(the)X
2148(error)X
2353(code)X
2554(length)X
2810(is)X
2904(zero.)X
3112(However)X
3468(the)X
3611(model)X
3866(code)X
576 1536(length)N
824(reaches)X
1114(its)X
1225(largest)X
1487(value,)X
1727(resulting)X
2063(in)X
2159(a)X
2225(larger)X
2458(total)X
2644(code)X
2838(length)X
3087(again.)X
3328(The)X
3493(minimum)X
3866(code)X
576 1696(length)N
819(occurred)X
1148(at)X
1234(extraction)X
1609(interval)X
1901(of)X
1996(8,)X
2084(which)X
3 f
10 s
576 3896(Figure)N
823(4.)X
1 f
923(Optimization)X
1365(correct)X
1609(ratio)X
11 s
576 4136(gives)N
796(98.2)X
984(percent)X
1277(correct)X
1555(classi\256cation.)X
2073(Figure)X
2338(4)X
2417(illustrates)X
2796(the)X
2939(fraction)X
3248(of)X
3356(correctly)X
3704(classi\256ed)X
576 4296(character)N
920(drawings)X
1264(for)X
1388(the)X
1518(training)X
1815(data.)X
776 4484(Whether)N
1109(the)X
1247(resulting)X
1586(`optimal')X
1945(model)X
2196(really)X
2427(performs)X
2774(better)X
3005(than)X
3187(the)X
3326(models)X
3612(in)X
3712(the)X
3851(same)X
576 4644(class,)N
802(the)X
943(knowledge)X
1362(bases)X
1585(established)X
2010(using)X
2234(different)X
2570(feature)X
2847(extraction)X
3232(intervals,)X
3590(is)X
3681(subject)X
3963(to)X
576 4804(testing)N
839(it)X
916(on)X
1031(new)X
1204(character)X
1553(drawings.)X
1924(For)X
2073(this)X
2228(purpose,)X
2555(the)X
2690(set)X
2815(of)X
2915(62)X
3030(handwritten)X
3478(characters)X
3862(were)X
576 4964(drawn)N
821(again)X
1038(by)X
1152(the)X
1286(same)X
1493(person)X
1753(who)X
1930(provided)X
2269(the)X
2403(initial)X
2636(data)X
2808(to)X
2902(build)X
3109(the)X
3242(knowledge)X
3653(base.)X
3856(Thus)X
576 5124(the)N
707(new)X
876(data)X
1046(can)X
1191(be)X
1297(considered)X
1701(to)X
1793(be)X
1899(from)X
2093(the)X
2225(same)X
2430(source)X
2683(as)X
2780(the)X
2912(previous)X
3239(data)X
3410(set.)X
3554(The)X
3715(new)X
3885(data)X
576 5284(were)N
781(classi\256ed)X
1143(by)X
1265(the)X
1407(system)X
1686(using)X
1911(the)X
2053(knowledge)X
2473(base)X
2663(built)X
2860(from)X
3065(the)X
3207(former)X
3480(data)X
3661(set)X
3793(of)X
3900(186)X
576 5444(character)N
923(drawings,)X
1292(based)X
1517(on)X
1630(different)X
1958(feature)X
2228(extraction)X
2607(intervals.)X
2959(The)X
3122(testing)X
3384(result)X
3606(is)X
3691(plotted)X
3963(in)X
576 5604(Figure)N
844(5)X
926(in)X
1032(terms)X
1265(of)X
1375(the)X
1520(fraction)X
1831(of)X
1941(correct)X
2222(classi\256cation)X
2720(\(CORRECT)X
3193(RATIO\))X
3527(versus)X
3788(feature)X
576 5764(extraction)N
983(interval)X
1307(\(SAMPLING)X
1838(INTERVAL\).)X
2404(It)X
2512(is)X
2625(interesting)X
3052(to)X
3175(see)X
3341(that)X
3528(100%)X
3788(correct)X
ep
%%Page: 36 36
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(36)X
2370(-)X
3 f
10 s
576 2776(Figure)N
823(5.)X
1 f
923(Test)X
1081(result)X
11 s
576 3016(classi\256cation)N
1059(occurred)X
1388(at)X
1474(feature)X
1740(extraction)X
2115(intervals)X
2441(5,)X
2529(6)X
2596(and)X
2746(7.)X
2835(These)X
3068(values)X
3316(of)X
3412(feature)X
3679(extraction)X
576 3176(intervals)N
903(are)X
1032(close)X
1235(to)X
1326(the)X
1456(optimized)X
1832(value)X
2045(8.)X
2155(At)X
2265(the)X
2395(low)X
2549(end)X
2698(of)X
2793(feature)X
3059(extraction)X
3434(interval)X
3726(scale)X
3924(the)X
576 3336(correct)N
845(classi\256cation)X
1331(drops,)X
1573(indicating)X
1952(disturbance)X
2384(caused)X
2649(by)X
2763(too)X
2902(much)X
3124(redundancy)X
3560(in)X
3655(the)X
3789(model.)X
576 3496(The)N
738(recommended)X
1262(working)X
1580(feature)X
1849(extraction)X
2227(interval)X
2522(is)X
2606(thus)X
2778(either)X
3004(7)X
3073(or)X
3171(8)X
3240(for)X
3366(this)X
3518(particular)X
3880(type)X
576 3656(of)N
672(character)X
1017(drawings.)X
1384(For)X
1529(more)X
1733(information)X
2173(on)X
2284(this)X
2435(research,)X
2772(see)X
2908([)X
2961(gao)X
3112(li)X
3186(\(preprint)X
3518(available)X
3861(from)X
576 3816(the)N
706(\256rst)X
865(author\).)X
3 f
576 4136(5.)N
686(Fisher's)X
1011(Maximum)X
1424(Likelihood)X
1854(Principle)X
1 f
576 4324(Rissanen)N
930([)X
995(universal)X
1354(prior)X
1561(has)X
1714(argued)X
1989(that)X
2158(Fisher's)X
2477(maximum)X
2872(likelihood)X
3267(principle)X
3617([)X
3682(Fisher)X
3938(])X
4003([)X
576 4484(Gauss)N
813(is)X
895(a)X
957(special)X
1225(case)X
1399(of)X
1495(the)X
1626(MDL)X
1844(principle.)X
2203(By)X
2329(our)X
2469(treatment)X
2826(of)X
2922(MDL)X
3140(it)X
3213(is)X
3295(therefore)X
3635(a)X
3697(more)X
3901(res-)X
576 4644(tricted)N
832(computable)X
1274(approximation)X
1819(to)X
1918(Solomonoff's)X
2432(induction.)X
2840(The)X
2 f
3007(Maximum)X
3388(Likelihood)X
1 f
3797(princi-)X
576 4804(ple)N
730(says)X
927(that)X
1106(given)X
1348(data)X
2 f
1540(D)X
1 f
1603(,)X
1670(one)X
1842(should)X
2122(use)X
2284(the)X
2437(hypothesis)X
2 f
2859(H)X
1 f
2967(that)X
3145(maximizes)X
2 f
3573(P)X
1 f
3641(\()X
2 f
3670(D)X
1 f
9 f
3761(|)X
2 f
3793(H)X
1 f
3863(\))X
3937(or,)X
576 4964(equivalently,)N
1058(minimizes)X
2 f
9 f
1450(-)X
1 f
1498(log)X
2 f
1611(P)X
1 f
1679(\()X
2 f
1708(D)X
1 f
9 f
1799(|)X
2 f
1831(H)X
1 f
1901(\),)X
1975(the)X
2106(\256rst)X
2266(term)X
2451(in)X
2543(of)X
2639(the)X
2770(MDL)X
2988(principle.)X
3369(We)X
3515(will)X
3677(use)X
2 f
3818(H)X
1 f
3905(and)X
9 f
576 5124(q)N
1 f
646(interchangeably)X
1233(because)X
9 f
1535(q)X
1 f
1605(is)X
1688(used)X
1873(by)X
1985(statisticians.)X
2444(What)X
2659(makes)X
2908(ML)X
3064(principle)X
3402(sound)X
3635(in)X
3727(statistics)X
576 5284(is)N
692(the)X
857(implicit)X
1191(assumption)X
1650(that)X
1840(each)X
2058(hypothesis)X
2 f
2492(H)X
1 f
2613(consists)X
2950(of)X
3081(a)X
3178(probability)X
3624(distribution)X
9 f
576 5444(q)N
2 f
9 f
622(=)X
1 f
670(\()X
9 f
699(q)X
8 s
1 f
745 5461(1)N
11 s
784 5444(,)N
835 5418(.)N
879(.)X
923(.)X
967 5444(,)N
9 f
996(q)X
2 f
8 s
1042 5461(k)N
1 f
11 s
1077 5444(\))N
1132(with)X
1315(the)X
1449(same)X
1656(number)X
2 f
1951(k)X
1 f
2016(of)X
2114(parameters,)X
2547(each)X
2733(parameter)X
9 f
3110(q)X
2 f
8 s
3156 5461(i)N
1 f
11 s
3206 5444(with)N
3388(\256xed)X
3589(precision.)X
3959(In)X
576 5604(other)N
781(words,)X
1041(in)X
1134(the)X
1266(probability)X
1678(distribution)X
2 f
2110(P)X
1 f
2178(\()X
2 f
2207(D)X
1 f
9 f
2298(|)X
2 f
2344(H)X
9 f
2428(=)X
1 f
9 f
2490(q)X
1 f
2536(\))X
2589(the)X
2721(number)X
2 f
3014(k)X
1 f
3077(of)X
3174(parameters)X
3584(of)X
9 f
3681(q)X
1 f
3727(,)X
3773(and)X
3924(the)X
576 5764(precision)N
922(of)X
1018(each)X
1202(of)X
1298(them,)X
1520(is)X
1602(the)X
1733(same)X
1937(for)X
2062(each)X
2 f
2246(H)X
1 f
2309(.)X
2376(Hence,)X
2645(one)X
2795(assumes)X
3110(that)X
3265(the)X
3395(descriptions)X
3843(of)X
2 f
3938(all)X
ep
%%Page: 37 37
bp
11 s 0 xH 0 xS 2 f
1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(37)X
2370(-)X
2 f
576 736(hypotheses)N
989(\(models)X
1 f
9 f
1294(q)X
1 f
1340(\))X
1396(are)X
1530(of)X
1630(equal)X
1848(length;)X
2121(that)X
2281(is,)X
2389(the)X
2524(complexity)X
2949(of)X
3049(the)X
3184(models)X
3467(is)X
3554(considered)X
3963(to)X
576 896(be)N
690(\256xed.)X
918(This)X
1105(is,)X
1216(obviously,)X
1616(a)X
2 f
1685(subjective)X
1 f
2068(assumption.)X
2522(In)X
2625(contrast,)X
2956(the)X
3094(MDL)X
3319(principle)X
3663(minimizes)X
576 1056(the)N
710(sum)X
883(of)X
2 f
9 f
982(-)X
1 f
1030(log)X
2 f
1143(P)X
1 f
1211(\()X
2 f
1240(D)X
1 f
9 f
1331(|)X
2 f
1363(H)X
1 f
1433(\))X
1488(and)X
2 f
9 f
1641(-)X
1 f
1689(log)X
2 f
1802(P)X
1 f
1870(\()X
2 f
1899(H)X
1 f
1969(\))X
2024(Intuitively,)X
2441(if)X
2521(one)X
2674(increases)X
3022(the)X
3157(description)X
3576(length)X
3824(of)X
3924(the)X
576 1216(hypothesis)N
2 f
978(H)X
1 f
1041(,)X
1088(it)X
1163(may)X
1340(\256t)X
1439(the)X
1572(data)X
1744(better)X
1970(and)X
2122(therefore)X
2464(decrease)X
2791(the)X
2924(description)X
3340(of)X
3437(data)X
3608(given)X
2 f
3828(H)X
1 f
3891(.)X
3959(In)X
576 1376(the)N
717(extreme)X
1034(case,)X
1240(if)X
1328(one)X
1489(encodes)X
1806(all)X
1929(the)X
2071(data)X
2252(information)X
2703(into)X
2875(the)X
3017(model)X
2 f
3272(H)X
1 f
3369(precisely,)X
2 f
3743(P)X
1 f
3811(\()X
2 f
3840(H)X
1 f
3910(\))X
3973(is)X
576 1536(minimized)N
978(and)X
2 f
9 f
1127(-)X
1 f
1189(log)X
2 f
1316(P)X
1 f
1384(\()X
2 f
1413(H)X
1 f
1483(\))X
1534(is)X
1615(maximized.)X
2074(In)X
2169(that)X
2324(case,)X
2519(no)X
2629(code)X
2817(is)X
2898(needed)X
3169(to)X
3260(describe)X
3575(the)X
3705(data;)X
3899(that)X
576 1696(is,)N
2 f
679(P)X
1 f
747(\()X
2 f
776(D)X
1 f
9 f
867(|)X
2 f
899(H)X
1 f
969(\))X
1020(is)X
1101(maximized)X
1516(\(equals)X
1792(1\))X
1887(and)X
2 f
9 f
2036(-)X
1 f
2084(log)X
2 f
2197(P)X
1 f
2265(\()X
2 f
2294(D)X
1 f
9 f
2385(|)X
2 f
2417(H)X
1 f
2487(\))X
2538(is)X
2619(minimized)X
3020(\(equals)X
3296(0\).)X
776 1884(On)N
909(the)X
1043(other)X
1250(hand,)X
1469(if)X
1549(one)X
1702(decreases)X
2064(the)X
2198(description)X
2616(length)X
2863(of)X
2 f
2962(H)X
1 f
3025(,)X
3073(then)X
3251(this)X
3405(may)X
3584(be)X
3694(penalized)X
576 2044(by)N
694(the)X
832(increasing)X
1224(description)X
1646(length)X
1896(of)X
1998(the)X
2135(data,)X
2333(given)X
2 f
2558(H)X
1 f
2621(.)X
2672(In)X
2774(the)X
2911(extreme)X
3224(case)X
3404(say,)X
2 f
3572(H)X
1 f
3664(is)X
3752(a)X
3820(trivial)X
576 2204(hypothesis)N
986(that)X
1152(contains)X
1479(nothing,)X
1804(then)X
1989(one)X
2149(needs)X
2382(0)X
2460(bits)X
2622(to)X
2725(describe)X
2 f
3052(H)X
1 f
3115(.)X
3193(But)X
3355(then,)X
3563(one)X
3724(gains)X
3944(no)X
576 2364(insight)N
839(of)X
934(data)X
1103(and)X
1252(has)X
1391(to)X
1482(`plainly')X
1808(describe)X
2123(the)X
2253(data)X
2422(without)X
2714(help)X
2888(from)X
3081(any)X
3230(hypothesis.)X
776 2552(Hence)N
1023(one)X
1173(may)X
1348(consider)X
1669(the)X
1800(MDL)X
2018(principle)X
2355(as)X
2451(a)X
2513(more)X
2717(general)X
2999(principle)X
3336(than)X
3511(the)X
3642(ML)X
3797(princi-)X
576 2712(ple)N
717(in)X
819(the)X
960(sense)X
1183(that)X
1349(it)X
1432(considers)X
1797(the)X
1938(trade-off)X
2277(between)X
2602(the)X
2742(complexity)X
3172(of)X
3277(the)X
3417(model)X
2 f
3670(H)X
1 f
3765(and)X
3924(the)X
576 2872(power)N
817(of)X
912(the)X
1042(model)X
1285(to)X
1376(describe)X
1692(the)X
1823(data)X
2 f
1993(D)X
1 f
2056(,)X
2101(whereas)X
2411(the)X
2542(ML)X
2697(principle)X
3034(does)X
3218(not)X
3354(take)X
3524(the)X
3655(hypothesis)X
576 3032(complexity)N
996(into)X
1156(account.)X
776 3220(Yet)N
930(the)X
1065(rationale)X
1401(behind)X
1668(the)X
1803(ML)X
1962(principle)X
2303(was)X
2467(to)X
2564(be)X
2675(objective)X
3027(by)X
3143(avoiding)X
3480(the)X
3616(`subjective')X
576 3380(assumption)N
1001(of)X
1097(the)X
1228(prior)X
1422(probability.)X
1877(The)X
2037(ML)X
2192(principle)X
2529(is)X
2611(equivalent)X
3002(with)X
3182(selecting)X
3519(the)X
3650(probabilis-)X
576 3540(tic)N
692(model)X
2 f
941(P)X
1 f
1009(\()X
2 f
1038(D)X
1 f
9 f
1129(|)X
1161(q)X
1 f
1207(\))X
1286(which)X
1529(permits)X
1822(the)X
1958(shortest)X
2260(ideal)X
2460(code)X
2654(length)X
2903(for)X
3033(the)X
3169(observed)X
3514(sequence,)X
3886(pro-)X
576 3700(vided)N
798(that)X
956(the)X
1089(model)X
1335(used)X
1521(in)X
1615(the)X
1748(encoding,)X
2 f
2118(i.e.)X
1 f
2226(,)X
2280(the)X
2413(parameter)X
9 f
2790(q)X
1 f
2861(is)X
2945(given,)X
3188(too.)X
3348(Thus,)X
3571(the)X
3704(ML)X
3861(prin-)X
576 3860(ciple)N
771(is)X
853(just)X
1004(a)X
1066(special)X
1334(case)X
1508(of)X
1604(the)X
1735(MDL)X
1953(principle)X
2290(under)X
2513(the)X
2644(assumption)X
3069(that)X
3226(hypotheses)X
3641(are)X
3772(equally)X
576 4020(likely)N
805(and)X
958(the)X
1092(number)X
1387(of)X
1486(parameters)X
1898(in)X
9 f
1993(q)X
1 f
2065(are)X
2198(\256xed)X
2400(and)X
2553(small)X
2771(\(so)X
2904(they)X
3082(do)X
3196(not)X
3335(make)X
2 f
3552(P)X
1 f
3620(\()X
2 f
3649(D)X
1 f
9 f
3740(|)X
3786(q)X
1 f
3832(\))X
2 f
9 f
3875(=)X
1 f
3937(1\).)X
576 4180(The)N
737(shortcoming)X
1202(of)X
1299(the)X
1431(ML)X
1587(principle)X
1925(is)X
2008(that)X
2165(it)X
2239(cannot)X
2498(handle)X
2757(the)X
2890(situation)X
3220(where)X
3459(we)X
3586(do)X
3699(not)X
3837(know)X
576 4340(the)N
707(number)X
999(\(and)X
1178(precision\))X
1553(of)X
1649(the)X
1780(parameters.)X
2211(For)X
2356(example,)X
2700(in)X
2792(the)X
2922(\256tting)X
3156(polynomial)X
3581(example,)X
3924(the)X
576 4500(ML)N
738(principle)X
1082(does)X
1273(not)X
1416(work)X
1626(well)X
1809(when)X
2030(the)X
2169(degree)X
2434(of)X
2538(the)X
2677(polynomial)X
3111(is)X
3201(not)X
3345(\256xed.)X
3574(On)X
3712(the)X
3851(other)X
576 4660(hand)N
769(the)X
899(MDL)X
1116(principle)X
1452(works)X
1688(naturally)X
2024(for)X
2148(this)X
2298(example.)X
3 f
576 4980(6.)N
686(Jaynes')X
991(Maximum)X
1404(Entropy)X
1739(Principle)X
1 f
776 5168(Rissanen)N
1116([)X
1167(universal)X
1512(prior)X
1705(and)X
1854(M.)X
1976(Feder)X
2199([)X
2251(Feder)X
2474(have)X
2663(shown)X
2915(that)X
3071(Jaynes')X
3357(Maximum)X
3748(Entropy)X
576 5328(\(ME\))N
797(principle)X
1142([)X
1202(Jaynes)X
1467(rationale)X
1806(maximum)X
2195(entropy)X
2494(])X
2553([)X
2612(Jaynes)X
2876(information)X
3323(inference)X
3680(])X
3739([)X
3798(Jaynes)X
576 5488(Probabilities)N
1053(can)X
1205(also)X
1377(be)X
1490(considered)X
1901(as)X
2004(a)X
2073(special)X
2348(case)X
2529(of)X
2632(the)X
2770(MDL)X
2995(principle.)X
3383(This)X
3570(is)X
3659(interesting)X
576 5648(since)N
791(it)X
875(is)X
968(known)X
1240(in)X
1342(statistics)X
1680(that)X
1846(there)X
2055(are)X
2195(a)X
2267(number)X
2569(of)X
2675(important)X
3052(applications)X
3512(where)X
3759(the)X
3900(ML)X
576 5808(principle)N
914(fails)X
1090(but)X
1227(where)X
1465(the)X
1597(maximum)X
1980(entropy)X
2274(formalism)X
2662(has)X
2804(been)X
2995(successful,)X
3403(and)X
3555(vice)X
3727(versa.)X
3959(In)X
ep
%%Page: 38 38
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(38)X
2370(-)X
576 736(order)N
787(to)X
882(apply)X
1104(Bayes')X
1373(theorem,)X
1709(we)X
1836(need)X
2027(to)X
2121(decide)X
2376(what)X
2572(the)X
2705(prior)X
2901(probability)X
2 f
3314(p)X
8 s
753(i)Y
11 s
9 f
3397 736(=)N
2 f
3459(P)X
1 f
3527(\()X
2 f
3556(H)X
8 s
3619 753(i)N
1 f
11 s
3644 736(\))N
3698(is)X
3782(subject)X
576 896(to)N
667(condition)X
2 f
8 s
2178 1226(i)N
1 f
16 s
9 f
2149 1162(S)N
2 f
11 s
2225 1136(p)N
8 s
1153(i)Y
1 f
11 s
2 f
9 f
2316 1136(=)N
1 f
2386(1,)X
576 1448(and)N
735(certain)X
1007(other)X
1220(constraints)X
1634(provided)X
1979(by)X
2099(empirical)X
2465(data)X
2644(or)X
2749(considerations)X
3290(of)X
3395(symmetry,)X
3803(proba-)X
576 1608(bilistic)N
841(laws,)X
1047(and)X
1197(so)X
1298(on.)X
1431(Usually)X
1728(these)X
1932(constraints)X
2337(are)X
2466(not)X
2601(suf\256cient)X
2951(to)X
3042(determine)X
3417(the)X
2 f
3547(p)X
8 s
1625(i)Y
1 f
11 s
3616 1608('s)N
3701(uniquely.)X
576 1772(Jaynes)N
840(proposed)X
1192(to)X
1291(use)X
1438(the)X
1576(estimated)X
1945(values)X
2 f
2200(p)X
1 f
12 s
2208 1764(\303)N
2 f
8 s
2244 1789(i)N
1 f
11 s
2299 1772(which)N
2544(satisfy)X
2804(said)X
2976(constraints)X
3388(and)X
3545(maximize)X
3924(the)X
576 1932(entropy)N
867(function)X
2 f
2029 2172(H)N
1 f
2 f
9 f
2121(=)X
1 f
2 f
9 f
2191(-)X
8 s
2 f
2282 2262(i)N
1 f
16 s
9 f
2253 2198(S)N
2 f
11 s
2343 2172(p)N
8 s
2189(i)Y
1 f
11 s
2426 2172(ln)N
2 f
2509(p)X
8 s
2189(i)Y
1 f
11 s
576 2484(subject)N
848(to)X
939(the)X
1069(constraints.)X
1517(This)X
1696(is)X
1777(called)X
2010(the)X
2 f
2140(maximum)X
2503(entropy)X
2794(\(ME\))X
1 f
3001(principle.)X
776 2672(We)N
922(now)X
1097(demonstrate)X
1552(the)X
1684(rationale)X
2017(behind)X
2281(the)X
2413(ME)X
2569(principle,)X
2929(its)X
3038(use,)X
3202(and)X
3354(its)X
3463(connection)X
3875(with)X
576 2832(the)N
718(MDL)X
947(principle)X
1294(following)X
1670(discussions)X
2104(in)X
2206([)X
2268(Jaynes)X
2535(rationale)X
2877(])X
2939([)X
3001(Feder)X
3234(])X
3296([)X
3358(Rissanen)X
3709(universal)X
576 2992(prior)N
772(Consider)X
1115(a)X
1179(random)X
1473(experiment)X
1895(with)X
2 f
2077(k)X
1 f
2141(possible)X
2455(outcomes)X
2818(in)X
2912(each)X
3098(trial,)X
3288(thus)X
2 f
3461(k)X
8 s
3510 2957(n)N
1 f
11 s
3575 2992(possible)N
3890(out-)X
576 3152(comes)N
825(in)X
2 f
918(n)X
1 f
986(trails.)X
1231(Let)X
2 f
1373(n)X
8 s
3169(i)Y
1 f
11 s
1466 3152(be)N
1572(the)X
1703(number)X
1995(of)X
2091(times)X
2306(the)X
2 f
2437(i)X
1 f
2462(th)X
2554(value)X
2768(appears)X
3059(in)X
3151(an)X
3257(outcome)X
2 f
3584(D)X
1 f
3670(of)X
2 f
3766(n)X
1 f
3833(trials.)X
576 3312(Let)N
716(frequency)X
2 f
1096(f)X
8 s
1121 3329(i)N
11 s
9 f
1160 3312(=)N
2 f
1222(n)X
8 s
3329(i)Y
11 s
1298 3312(/)N
1330(n)X
1 f
(,)S
2 f
1418(i)X
9 f
1464(=)X
1 f
1526(1,)X
1613(2,)X
1686(...,)X
2 f
1781(k)X
1 f
1820(.)X
1864(The)X
2 f
2023(entropy)X
1 f
2314(of)X
2409(outcome)X
2 f
2735(D)X
1 f
2820(is:)X
2 f
1809 3594(H)N
1 f
1886(\()X
2 f
1922(f)X
1 f
8 s
1957 3611(1)N
11 s
1996 3594(,)N
2040(.)X
2084(.)X
2128(.)X
2172(,)X
2 f
2215(f)X
8 s
2240 3611(k)N
1 f
11 s
2275 3594(\))N
2 f
9 f
2326(=)X
1 f
2 f
9 f
2396(-)X
8 s
2 f
2458 3684(i)N
9 f
2486(=)X
1 f
2521(1)X
16 s
9 f
2467 3620(S)N
2 f
8 s
2491 3511(k)N
11 s
2573 3594(f)N
8 s
2598 3611(i)N
1 f
11 s
2637 3594(ln)N
2 f
2727(f)X
8 s
2752 3611(i)N
11 s
2777 3594(.)N
1 f
3886(\(10\))X
576 3906(Let)N
716(there)X
914(be)X
2 f
1019(m)X
1103(<)X
1176(k)X
1 f
1237(linearly)X
1529(independent)X
1982(constraints)X
2386(of)X
2481(the)X
2611(form)X
2 f
8 s
1016 4278(i)N
9 f
1044(=)X
1 f
1079(1)X
16 s
9 f
1025 4214(S)N
2 f
8 s
1049 4105(k)N
11 s
1124 4188(a)N
8 s
1173 4205(ji)N
11 s
1237 4188(f)N
8 s
1262 4205(i)N
1 f
11 s
2 f
9 f
1309 4188(=)N
1 f
2 f
1379(d)X
8 s
1428 4205(j)N
1 f
11 s
1453 4188(,)N
1526(1)X
9 f
1584(\243)X
2 f
1653(j)X
1 f
9 f
1699(\243)X
2 f
1761(m)X
1 f
1824(,)X
1882(and)X
3886(\(11\))X
2 f
8 s
1016 4632(i)N
9 f
1044(=)X
1 f
1079(1)X
16 s
9 f
1025 4568(S)N
2 f
8 s
1049 4459(k)N
11 s
1131 4542(f)N
8 s
1156 4559(i)N
1 f
11 s
2 f
9 f
1203 4542(=)N
1 f
1273(1)X
3886(\(12\))X
576 4854(where)N
823(the)X
964(set)X
3 f
1095(D)X
2 f
9 f
1172(=)X
2 f
1234({d)X
1 f
8 s
1323 4871(1)N
11 s
1362 4854(,)N
2 f
1391(...)X
1 f
(,)S
2 f
1486(d)X
8 s
4871(m)Y
11 s
1583 4854(})N
1 f
1651(is)X
1743(related)X
2017(to)X
2120(the)X
2262(observed)X
2613(data,)X
2816(measuring)X
3217(as)X
3324(it)X
3408(were)X
2 f
3612(m)X
1 f
3709(`physical)X
576 5014(quantities')N
971(subject)X
1243(to)X
1334(the)X
1464(matrix)X
2 f
1717(A)X
9 f
1792(=)X
2 f
1854({a)X
8 s
1938 5031(ji)N
11 s
1981 5014(})N
1 f
2016(.)X
3 f
776 5202(Example.)N
1 f
1183(Consider)X
1532(a)X
1602(loaded)X
1868(die,)X
2 f
2029(k)X
9 f
2089(=)X
1 f
2151(6.)X
2248(If)X
2337(we)X
2470(do)X
2589(not)X
2733(have)X
2930(any)X
3088(information)X
3536(about)X
3763(the)X
3902(die,)X
576 5362(then)N
778(using)X
1019(the)X
1177(Epicurus')X
1569(multiple)X
1915(explanation)X
2377(principle,)X
2763(we)X
2915(may)X
3117(assume)X
3426(that)X
2 f
3609(p)X
8 s
5379(i)Y
11 s
9 f
3692 5362(=)N
1 f
3754(1)X
2 f
3805(/)X
1 f
3837(6)X
3930(for)X
2 f
576 5540(i)N
9 f
622(=)X
1 f
684(1,)X
757(...,)X
852(6.)X
970(This)X
1157(actually)X
1467(coincides)X
1830(with)X
2017(the)X
2155(ME)X
2318(principle,)X
2685(since)X
2 f
2897(H)X
1 f
2974(\()X
2 f
3003(p)X
1 f
8 s
3057 5557(1)N
11 s
3096 5540(,)N
2 f
3125(...)X
1 f
(,)S
2 f
3220(p)X
1 f
8 s
3274 5557(6)N
11 s
3313 5540(\))N
2 f
9 f
3356(=)X
1 f
16 s
9 f
3418 5566(S)N
2 f
8 s
3499 5584(i)N
9 f
3527(=)X
1 f
3562(1)X
3499 5507(6)N
2 f
11 s
3615 5540(p)N
8 s
5557(i)Y
1 f
11 s
3698 5540(ln)N
2 f
3781(p)X
8 s
5557(i)Y
1 f
11 s
3881 5540(sub-)N
576 5700(ject)N
735(to)X
835(\(12\))X
1012(achieves)X
1346(maximum)X
1735(value)X
1956(ln6)X
2 f
9 f
2083(=)X
1 f
2145(1.7917595)X
2549(for)X
2 f
2681(p)X
8 s
5717(i)Y
11 s
9 f
2764 5700(=)N
1 f
2826(1)X
2 f
2877(/)X
1 f
2909(6)X
2983(for)X
3115(all)X
2 f
3234(i)X
1 f
3259(.)X
3333(Now)X
3533(suppose)X
3846(some)X
576 5860(experiments)N
1041(on)X
1163(the)X
1305(die)X
1447(have)X
1647(been)X
1847(performed,)X
2269(and)X
2430(it)X
2514(is)X
2607(observed)X
2958(that)X
3125(the)X
3267(die)X
3409(is)X
3502(biased)X
3762(and)X
3924(the)X
ep
%%Page: 39 39
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(39)X
2370(-)X
576 736(average)N
871(throw)X
1098(gives)X
1306(4.5.)X
1460(That)X
1644(is,)X
2 f
8 s
2080 1108(i)N
9 f
2108(=)X
1 f
2143(1)X
16 s
9 f
2089 1044(S)N
8 s
1 f
2111 935(6)N
2 f
11 s
2188 1018(i)N
2234(p)X
8 s
1035(i)Y
1 f
11 s
2 f
9 f
2325 1018(=)N
1 f
2395(4.5)X
2 f
(.)S
1 f
576 1330(In)N
685(terms)X
917(of)X
1026(Equation)X
1381(\(11\),)X
1585(we)X
1723(have)X
2 f
1925(m)X
9 f
2009(=)X
1 f
2071(1,)X
3 f
2173(D)X
2 f
9 f
2250(=)X
2 f
2312({)X
1 f
2347(4.5)X
2 f
(})S
1 f
2492(,)X
2551(and)X
2 f
2715(a)X
8 s
2764 1347(j)N
1 f
2792(1)X
2 f
11 s
9 f
2845 1330(=)N
1 f
2907(\(1,)X
3009(2,)X
3082(3,)X
3155(4,)X
3228(5,)X
3301(6\).)X
3455(Maximizing)X
3924(the)X
576 1490(expression)N
974(in)X
1065(Equation)X
1406(\(10\),)X
1596(subject)X
1868(to)X
1959(constraints)X
2363(\(11\))X
2531(and)X
2680(\(12\))X
2848(gives)X
3056(estimates:)X
2 f
1758 1734(p)N
1 f
12 s
1766 1726(\303)N
2 f
8 s
1802 1751(i)N
1 f
11 s
2 f
9 f
1849 1734(=)N
1 f
2 f
1919(e)X
8 s
9 f
1968 1699(-)N
1 f
9 f
2003(l)X
2 f
2038(i)X
1 f
11 s
2063 1734(\()N
16 s
9 f
2092 1760(S)N
2 f
11 s
2168 1734(e)N
8 s
9 f
2217 1699(-)N
1 f
9 f
2252(l)X
2 f
2287(i)X
1 f
11 s
2312 1734(\))N
2 f
8 s
9 f
2341 1699(-)N
1 f
2376(1)X
11 s
2415 1734(,)N
2488(1)X
2 f
9 f
2546(=)X
1 f
2608(1,)X
2681(...,)X
2776(6,)X
576 1986(where)N
9 f
829(l)X
2 f
9 f
891(=)X
953(-)X
1 f
1001(0.37105.)X
1348(Hence)X
1611(\()X
2 f
1640(p)X
1 f
12 s
1648 1978(\303)N
8 s
1694 2003(1)N
11 s
1733 1986(,)N
2 f
1762(...)X
1 f
(,)S
2 f
1857(p)X
1 f
12 s
1865 1978(\303)N
8 s
1911 2003(6)N
11 s
1950 1986(\))N
2 f
9 f
1993(=)X
1 f
2080(\(0.0543,)X
2394(0.0788,)X
2679(0,)X
2752(1142,)X
2971(0.1654,)X
3256(0.2398,)X
3541(0.3475\).)X
3895(The)X
576 2150(maximized)N
992(entropy)X
2 f
1284(H)X
1 f
1361(\()X
2 f
1390(p)X
1 f
12 s
1398 2142(\303)N
8 s
1444 2167(1)N
11 s
1483 2150(,)N
1534 2124(.)N
1578(.)X
1622(.)X
1666 2150(,)N
2 f
1695(p)X
1 f
12 s
1703 2142(\303)N
8 s
1749 2167(6)N
11 s
1788 2150(\))N
1840(equals)X
2088(1.61358.)X
2441(How)X
2634(dependable)X
3058(is)X
3139(the)X
3269(ME)X
3423(principle?)X
3798(Jaynes)X
576 2310(has)N
717(proven)X
985(an)X
1092(`entropy)X
1414(concentration)X
1918(theorem')X
2260(which,)X
2521(for)X
2647(example,)X
2992(implies)X
3277(that)X
3434(in)X
3527(an)X
3635(experiment)X
576 2470(of)N
2 f
679(N)X
9 f
759(=)X
1 f
821(1000)X
1027(trails,)X
1256(99.99%)X
1557(of)X
1660(all)X
1779(outcomes)X
2147(satisfying)X
2520(the)X
2658(constraints)X
3070(of)X
3173(Equations)X
3555(\(11\))X
3730(and)X
3886(\(12\))X
576 2630(have)N
764(entropy)X
1693 2922(1.602)N
9 f
1913(\243)X
2 f
1983(H)X
1 f
2060(\()X
2 f
2135 2983(n)N
2111 2852(n)N
1 f
8 s
2165 2869(1)N
11 s
10 f
2097 2896(h)N
2129(hh)X
1 f
2225 2922(,)N
2269(.)X
2313(.)X
2357(.)X
2401(,)X
2 f
2483 2983(n)N
2459 2852(n)N
1 f
8 s
2513 2869(6)N
11 s
10 f
2445 2896(h)N
2477(hh)X
1 f
2573 2922(\))N
9 f
2624(\243)X
1 f
2694(1.614)X
2 f
(.)S
1 f
776 3234(Now)N
968(we)X
1092(turn)X
1257(to)X
1349(the)X
1480(MDL)X
1698(principle)X
2035(to)X
2127(deal)X
2297(with)X
2477(the)X
2608(same)X
2812(problem.)X
3173(The)X
3333(following)X
3699(argument)X
576 3394(can)N
730(be)X
845(derived)X
1141(from)X
1344(probabilistic)X
1817(assumptions.)X
2306(But)X
2465(Kolmogorov)X
2946([)X
3006(Kolmogorov)X
3487(logical)X
3759(founda-)X
576 3554(tions)N
783(1969)X
994(])X
1058([)X
1122(Kolmogorov)X
1607(Combinatorial)X
2153(foundations)X
2604(1983)X
2815(advocated)X
3207(a)X
3281(purely)X
3541(combinatorial)X
576 3714(approach,)N
986(such)X
1212(as)X
1350(we)X
1517(give)X
1734(below,)X
2036(which)X
2316(does)X
2542(not)X
2720(need)X
2951(any)X
3143(such)X
3369(assumptions.)X
3914(Let)X
9 f
576 3874(q)N
2 f
9 f
636(=)X
1 f
698(\()X
2 f
727(p)X
1 f
8 s
781 3891(1)N
11 s
820 3874(,)N
871 3848(.)N
915(.)X
959(.)X
1003 3874(,)N
2 f
1032(p)X
8 s
3891(k)Y
1 f
11 s
1111 3874(\))N
1162(be)X
1267(the)X
1397(actual)X
1630(prior)X
1824(distribution)X
2255(of)X
2351(a)X
2413(random)X
2705(variable.)X
3034(We)X
3179(perform)X
3485(a)X
3547(sequence)X
3892(of)X
2 f
3988(n)X
1 f
576 4034(independent)N
1032(trials.)X
1278(Kolmogorov)X
1753(observed)X
2095(that)X
2253(the)X
2386(real)X
2543(substance)X
2910(of)X
3007(Formula)X
3330(\(10\))X
3500(is)X
3583(that)X
3740(we)X
3866(need)X
576 4194(approximately)N
2 f
1111(n)X
1176(H)X
1 f
1253(\()X
9 f
1282(q)X
1 f
1328(\))X
1383(bits)X
1537(to)X
1632(record)X
1882(the)X
2016(sequence)X
2364(of)X
2 f
2463(n)X
1 f
2533(outcomes.)X
2919(Namely,)X
3246(it)X
3322(suf\256ces)X
3616(to)X
3711(state)X
3899(that)X
576 4354(each)N
764(outcome)X
1095(appeared)X
2 f
1439(n)X
1 f
8 s
1493 4371(1)N
11 s
1532 4354(,)N
2 f
1561(...)X
1 f
(,)S
2 f
1656(n)X
8 s
4371(k)Y
1 f
11 s
1762 4354(times,)N
2003(respectively,)X
2478(and)X
2632(afterwards)X
3029(give)X
3208(the)X
3343(index)X
3565(of)X
3664(which)X
3905(one)X
576 4514(of)N
671(the)X
2 f
1786 4789(C)N
1 f
1859(\()X
2 f
1888(n)X
1 f
8 s
1942 4806(1)N
11 s
1981 4789(,)N
2032 4763(.)N
2076(.)X
2120(.)X
2164 4789(,)N
2 f
2193(n)X
8 s
4806(k)Y
1 f
11 s
2272 4789(\))N
2 f
9 f
2323(=)X
1 f
2 f
2415 4850(n)N
1 f
8 s
2469 4867(1)N
11 s
2508 4850(!)N
2559 4824(.)N
2603(.)X
2647(.)X
2 f
2691 4850(n)N
8 s
4867(k)Y
1 f
11 s
2770 4850(!)N
2 f
2564 4737(n)N
1 f
2622(!)X
10 f
2402 4763(h)N
2417(hhhhhhhhh)X
1 f
576 5090(possible)N
887(sequences)X
2 f
1265(D)X
1 f
1350(of)X
2 f
1445(n)X
1 f
1511(outcomes)X
1871(actually)X
2173(took)X
2352(place.)X
2604(For)X
2748(this)X
2898(no)X
3008(more)X
3211(than)X
2 f
1568 5330(k)N
1 f
1628(log)X
2 f
1755(n)X
9 f
1820(+)X
1 f
1882(log)X
2 f
2009(C)X
1 f
2082(\()X
2 f
2111(n)X
1 f
8 s
2165 5347(1)N
11 s
2204 5330(,)N
2255 5304(.)N
2299(.)X
2343(.)X
2387 5330(,)N
2 f
2416(n)X
8 s
5347(k)Y
1 f
11 s
2495 5330(\))N
2 f
9 f
2538(+)X
2 f
2600(O)X
1 f
2677(\(log)X
2833(log)X
2 f
2960(n)X
1 f
3011(\))X
576 5570(bits)N
750(are)X
903(needed.)X
1243(The)X
1427(\256rst)X
1611(term)X
1820(corresponds)X
2291(to)X
2 f
9 f
2407(-)X
1 f
2469(log)X
2 f
2596(P)X
1 f
2664(\()X
9 f
2693(q)X
1 f
2739(\),)X
2837(the)X
2992(second)X
3283(term)X
3492(corresponds)X
3963(to)X
2 f
9 f
576 5730(-)N
1 f
638(log)X
2 f
765(P)X
1 f
833(\()X
2 f
862(D)X
1 f
9 f
946(|)X
992(q)X
1 f
1038(\),)X
1119(and)X
1276(the)X
1414(third)X
1611(term)X
1803(represents)X
2189(the)X
2327(cost)X
2498(of)X
2600(encoding)X
2952(separators)X
3337(between)X
3659(the)X
3796(indivi-)X
576 5890(dual)N
752(items.)X
1012(Using)X
1246(Stirling's)X
1600(approximation)X
2140(for)X
2267(the)X
2400(factorial)X
2719(function,)X
3060(we)X
3187(\256nd)X
3349(that)X
3507(for)X
3634(large)X
2 f
3835(n)X
1 f
3904(this)X
ep
%%Page: 40 40
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(40)X
2370(-)X
576 736(is)N
657(approximately)X
2 f
1576 1028(n)N
1 f
1641(\()X
2 f
9 f
1684(-)X
8 s
2 f
1746 1118(i)N
9 f
1774(=)X
1 f
1809(1)X
16 s
9 f
1755 1054(S)N
2 f
8 s
1779 945(k)N
11 s
1888 1089(n)N
1876 958(n)N
8 s
975(i)Y
1 f
11 s
10 f
1862 1002(h)N
1870(hh)X
1 f
1980 1028(log)N
2 f
2141 1089(n)N
2129 958(n)N
8 s
975(i)Y
1 f
11 s
10 f
2115 1002(h)N
2123(hh)X
1 f
2219 1028(\))N
2 f
9 f
2270(=)X
1 f
2 f
2340(n)X
2405(H)X
1 f
2482(\()X
2 f
2557 1089(n)N
2533 958(n)N
1 f
8 s
2587 975(1)N
11 s
10 f
2519 1002(h)N
2551(hh)X
1 f
2647 1028(,)N
2698 1002(.)N
2742(.)X
2786(.)X
2830 1028(,)N
2 f
2898 1089(n)N
2881 958(n)N
8 s
975(k)Y
1 f
11 s
10 f
2867 1002(h)N
2885(hh)X
1 f
2981 1028(\))N
2 f
3010(.)X
1 f
576 1340(Since)N
2 f
796(k)X
1 f
859(and)X
2 f
1010(n)X
1 f
1078(are)X
1209(\256xed,)X
1431(the)X
1563(least)X
1749(upper)X
1973(bound)X
2217(on)X
2329(the)X
2461(minimum)X
2831(description)X
3248(length)X
3494(for)X
3621(an)X
3729(arbitrary)X
576 1500(sequence)N
938(of)X
2 f
1051(n)X
1 f
1135(outcomes)X
1513(under)X
1753(certain)X
2033(given)X
2269(constraints)X
3 f
2690(D)X
1 f
2792(is)X
2890(found)X
3134(by)X
3261(maximizing)X
3723(the)X
3870(term)X
576 1660(log)N
2 f
703(C)X
1 f
776(\()X
2 f
805(n)X
1 f
8 s
859 1677(1)N
11 s
898 1660(,)N
2 f
927(...)X
1 f
(,)S
2 f
1022(n)X
8 s
1677(k)Y
1 f
11 s
1101 1660(\))N
1152(subject)X
1424(to)X
1515(said)X
1679(constraints.)X
2127(This)X
2306(is)X
2387(equivalent)X
2777(to)X
2869(maximizing)X
3315(the)X
3446(entropy)X
3738(function)X
576 1820(\(10\))N
749(under)X
976(constraints)X
3 f
1385(D)X
1 f
1448(.)X
1519(\(Such)X
1751(constraints)X
2160(may)X
2339(be)X
2449(derived,)X
2762(for)X
2891(example,)X
3239(from)X
3436(the)X
3570(laws)X
3757(of)X
3856(large)X
576 1980(numbers:)N
928(in)X
1021(case)X
1196(of)X
1293(independent)X
1748(experiments)X
2203(with)X
2385(a)X
2449(probability)X
2862(distribution)X
9 f
3295(q)X
1 f
3341(,)X
3388(we)X
3515(have)X
2 f
3706(n)X
8 s
1997(i)Y
11 s
3782 1980(/)N
3814(n)X
9 f
3879(~)X
2 f
3941(p)X
8 s
1997(i)Y
1 f
11 s
4010 1980(,)N
576 2140(and)N
725(we)X
849(have)X
1037(a)X
1098(certain)X
1360(rate)X
1514(of)X
1609(convergence)X
2075(with)X
2254(certain)X
2516(probability.\))X
3 f
576 2460(7.)N
686(Valiant)X
987(Style)X
1195(Deductive)X
1593(Learning)X
1 f
576 2648(Can)N
740(we)X
864(make)X
1077(Gold-style)X
1471(learning)X
1782(feasible?)X
2139(According)X
2532(to)X
2624(commonly)X
3025(accepted)X
3356(views)X
3584(in)X
3676(the)X
3807(theory)X
576 2808(of)N
673(computation,)X
1161(this)X
1313(means)X
1562(that)X
1718(the)X
1849(learning)X
2161(algorithm)X
2528(should)X
2786(run)X
2926(in)X
3018(polynomial)X
3444(time)X
3625(-)X
3677(and)X
3827(hence)X
576 2968(also)N
746(use)X
891(but)X
1032(a)X
1099(polynomial)X
1530(number)X
1827(of)X
1928(examples.)X
2311(The)X
2476(latter)X
2686(condition)X
3048(necessarily)X
3467(implies)X
3757(that)X
3919(not)X
576 3128(all)N
691(examples)X
1049(in)X
1143(an)X
1251(in\256nite)X
1527(domain)X
1817(can)X
1964(turn)X
2131(up.)X
2288(Hence)X
2537(we)X
2664(need)X
2855(to)X
2949(assume)X
3233(a)X
3297(mechanism)X
3724(for)X
3851(mak-)X
576 3288(ing)N
719(a)X
789(selection)X
1134(of)X
1238(examples.)X
1624(A)X
1718(deterministic)X
2211(selection)X
2556(\256xes)X
2753(the)X
2892(sequence)X
3245(of)X
3349(examples)X
3713(drawn)X
3963(in)X
576 3448(advance,)N
918(hence)X
1155(we)X
1289(would)X
1541(like)X
1706(to)X
1807(assume)X
2098(that)X
2263(examples)X
2627(are)X
2765(drawn)X
3015(from)X
3217(some)X
3434(distribution.)X
3895(The)X
576 3608(idea)N
753(in)X
852(Gold's)X
1121(approach)X
1473(that)X
1636(an)X
1749(inference)X
2106(algorithm)X
2480(should)X
2745(work)X
2955(for)X
3088(all)X
3208(sequences)X
3595(of)X
3699(examples)X
576 3768(then)N
750(translates)X
1105(to)X
1196(the)X
1326(idea)X
1495(that)X
1650(the)X
1780(learning)X
2091(algorithm)X
2457(should)X
2714(work)X
2916(for)X
3040(all)X
3151(distributions.)X
776 3956(The)N
945(second)X
1221(unavoidable)X
1684(modi\256cation)X
2163(of)X
2269(the)X
2410(common)X
2752(approach)X
3107(in)X
3209(statistical)X
3577(inference,)X
3959(or)X
576 4116(recursion)N
931(theoretical)X
1332(learning,)X
1671(imposed)X
1998(by)X
2114(the)X
2250(feasability)X
2646(constraint,)X
3044(is)X
3131(as)X
3232(follows.)X
3568(In)X
3668(traditional)X
576 4276(inference)N
925(we)X
1049(want)X
1242(to)X
1333(learn)X
1531(a)X
1592(concept)X
1888(precisely)X
2228(in)X
2319(the)X
2450(limit.)X
2686(The)X
2846(feasibility)X
3223(restriction)X
3604(to)X
3696(a)X
3758(polyno-)X
576 4436(mial)N
769(algorithm)X
1147(precludes)X
1518(the)X
1660(precise)X
1943(learning)X
2266(of)X
2373(nontrivial)X
2751(concepts,)X
3115(and)X
3276(therefore)X
3627(we)X
3763(have)X
3963(to)X
576 4596(relax)N
774(precision)X
1119(to)X
1210(within)X
1458(a)X
1519(certain)X
1781(error.)X
1995(This)X
2174(corresponds)X
2620(with)X
2799(natural)X
3066(learning,)X
3400(where)X
3637(it)X
3710(is)X
3792(impor-)X
576 4756(tant)N
731(that)X
886(learning)X
1197(is)X
1278(fast,)X
1449(and)X
1598(it)X
1670(suf\256ces)X
1960(to)X
2051(learn)X
2249(approximately.)X
776 4944(We)N
927(have)X
1122(now)X
1303(arrived)X
1582(at)X
1676(Valiant's)X
2029(proposal:)X
2387(a)X
2456(learning)X
2775(theory,)X
3052(where)X
3296(one)X
3453(wants)X
3688(to)X
3787(learn)X
3993(a)X
576 5104(concept)N
875(with)X
1057(high)X
1238(propability,)X
1672(in)X
1765(polynomial)X
2192(time,)X
2396(and)X
2547(a)X
2610(polynomial)X
3037(number)X
3330(of)X
3427(examples,)X
3806(within)X
576 5264(a)N
650(certain)X
925(error,)X
1152(under)X
1387(all)X
1512(distributions)X
1990(on)X
2114(the)X
2258(examples.)X
2649(The)X
2822(additional)X
3212(computational)X
3754(require-)X
576 5424(ments)N
818(are)X
956(orthogonal)X
1369(to)X
1469(the)X
1608(usual)X
1825(concerns)X
2168(in)X
2268(inference,)X
2648(and)X
2805(result)X
3031(in)X
3130(a)X
3199(distinctly)X
3559(novel)X
3785(theory.)X
576 5584(However,)N
941(there)X
1139(are)X
1268(at)X
1354(least)X
1538(two)X
1692(problems)X
2042(with)X
2221(it:)X
776 5772(\(1\))N
909(Under)X
1159(all)X
1279(distributions,)X
1774(many)X
2001(concept)X
2306(classes,)X
2603(including)X
2968(some)X
3186(seemingly)X
3581(simple)X
3849(ones,)X
ep
%%Page: 41 41
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(41)X
2370(-)X
576 736(are)N
728(not)X
886(known)X
1170(to)X
1284(be)X
1412(polynomially)X
1929(learnable)X
2297(or)X
2415(known)X
2698(not)X
2855(to)X
2968(be)X
3095(polynomially)X
3611(learnable)X
3978(if)X
2 f
576 896(NP)N
9 f
696(\271)X
2 f
744(RP)X
1 f
852(,)X
896(although)X
1227(some)X
1435(concept)X
1731(classes)X
1997(are)X
2126(polynomially)X
2620(learnable)X
2965(under)X
3187(some)X
3395(\256xed)X
3593(distribution.)X
776 1084(\(2\))N
903(In)X
1001(real)X
1158(life)X
1301(situations,)X
1687(it)X
1762(is)X
1846(sometimes)X
2249(impossible)X
2657(to)X
2751(sample)X
3026(according)X
3398(to)X
3492(underlying)X
3900(dis-)X
576 1244(tributions.)N
776 1432(Item)N
964(\(1\))X
1092(is)X
1177(counterintuitive)X
1763(for)X
1891(a)X
1956(proposed)X
2304(theory)X
2555(of)X
2654(machine)X
2980(learning;)X
3321(in)X
3417(fact)X
3576(it)X
3653(shows)X
3899(that)X
576 1592(Valiant's)N
929(initiall)X
1162(y)X
1236(proposed)X
1588(requirements)X
2077(for)X
2208(learning)X
2526(are)X
2662(too)X
2804(strong.)X
3097(In)X
3199(practice,)X
3529(we)X
3660(usually)X
3944(do)X
576 1752(not)N
712(have)X
901(to)X
993(make)X
1207(such)X
1391(a)X
1453(general)X
1735(assumption.)X
2204(Due)X
2373(to)X
2466(this)X
2618(reason)X
2871(several)X
3144(authors)X
3427(have)X
3617(proposed)X
3963(to)X
576 1912(study)N
799(Valiant)X
1091(learning)X
1412(under)X
1644(\256xed)X
1851(distributions.)X
2346(Then)X
2558(some)X
2775(previously)X
3178(\(polynomially\))X
3739(unlearn-)X
576 2072(able)N
746(classes)X
1013(become)X
1310(learnable.)X
1700(For)X
1845(instance,)X
2179(the)X
2310(class)X
2505(of)X
9 f
2602(m)X
2 f
9 f
2653(-)X
2 f
2701(DNF)X
1 f
(-formulae)S
3248(is)X
3331(polynomially)X
3827(learn-)X
576 2232(able)N
747(under)X
971(the)X
1103(uniform)X
1411(distribution.)X
1887(However,)X
2254(the)X
2386(assumption)X
2812(of)X
2 f
2909(any)X
3060(special)X
1 f
3333(distribution)X
3764(is)X
3846(obvi-)X
576 2392(ously)N
789(too)X
924(restrictive)X
1299(and)X
1448(not)X
1583(practically)X
1978(interesting.)X
3 f
576 2712(7.1.)N
752(A)X
837(New)X
1024(Approach)X
1 f
576 2900(In)N
675([)X
730(Li)X
835(SIAM)X
1080(Simple)X
1357(Concepts)X
1711(we)X
1839(proposed)X
2187(to)X
2282(study)X
2499(Valiant-style)X
2981(learning)X
3296(under)X
2 f
3522(all)X
3643(simple)X
3900(dis-)X
576 3060(tributions)N
1 f
920(,)X
991(which)X
1254(properly)X
1600(include)X
2 f
1908(all)X
1 f
2050(computable)X
2510(and)X
2685(semi-computable)X
3341(distributions.)X
3875(This)X
576 3220(allows)N
846(us)X
964(to)X
1073(systematically)X
1620(develop)X
1940(a)X
2020(theory)X
2286(of)X
2400(learning)X
2730(for)X
2 f
2873(simple)X
3144(concepts)X
1 f
3493(that)X
3667(intuitivel)X
3988(y)X
576 3380(should)N
838(be)X
948(polynomially)X
1447(learnable.)X
1819(To)X
1944(stress)X
2166(this)X
2321(point:)X
2555(maybe)X
2817(it)X
2894(is)X
2980(too)X
3120(much)X
3343(to)X
3439(ask)X
3583(to)X
3679(be)X
3789(able)X
3963(to)X
576 3540(learn)N
784(all)X
905(\256nite)X
1119(automata)X
1475(fast)X
1634(\(humans)X
1969(cannot)X
2236(either\),)X
2521(but)X
2667(surely)X
2915(we)X
3050(ought)X
3284(to)X
3386(be)X
3502(able)X
3682(to)X
3784(learn)X
3993(a)X
2 f
576 3700(suf\256ciently)N
983(simple)X
1 f
1237(\256nite)X
1443(automaton)X
1839(fast)X
1989(\(as)X
2114(humans)X
2411(can\).)X
2629(Previous)X
2960(approaches)X
3378(looked)X
3641(at)X
3728(syntacti-)X
576 3860(cally)N
783(described)X
1155(classes)X
1434(of)X
1542(concepts.)X
1907(We)X
2064(introduce)X
2432(the)X
2575(idea)X
2757(of)X
2865(the)X
3008(restriction)X
3401(of)X
3510(a)X
3585(syntactically)X
576 4020(described)N
946(class)X
1150(of)X
1256(concepts)X
1597(to)X
1699(the)X
1840(concepts)X
2181(that)X
2347(are)X
2486(simple)X
2754(in)X
2855(the)X
2995(sense)X
3217(of)X
3322(having)X
3594(low)X
3758(Kolmo-)X
576 4180(gorov)N
803(complexity.)X
1245(This)X
1424(will)X
1584(cover)X
1802(most)X
1997(intuitive)X
2316(notions)X
2599(of)X
2695(simplicit)X
3006(y.)X
3095(Our)X
3254(other)X
3458(restriction,)X
3861(from)X
576 4340(distribution-free)N
1185(learning)X
1510(to)X
1615(simple-distributi)X
2200(on-free)X
2489(learning)X
2814(is)X
2909(also)X
3087(not)X
3236(much)X
3468(of)X
3577(a)X
3652(restriction.)X
576 4500(Already)N
888(the)X
1025(computable)X
1466(distributions)X
1937(include)X
2 f
2226(all)X
1 f
2349(distributions)X
2820(we)X
2951(have)X
3146(a)X
3214(name)X
3435(for,)X
3589(like)X
3752(the)X
3890(uni-)X
576 4660(form)N
777(distribution,)X
1237(normal)X
1517(distribution,)X
1977(geometric)X
2360(distribution,)X
2820(Poisson)X
3124(distribution)X
3562(-)X
3621(so)X
3729(the)X
3866(even)X
576 4820(wider)N
798(class)X
991(of)X
1086(simple)X
1344(distributions)X
1808(ought)X
2031(to)X
2122(cover)X
2339(everything)X
2738(practically)X
3133(interesting.)X
776 5008(It)N
857(is)X
943(an)X
1053(integral)X
1350(part)X
1514(of)X
1614(the)X
1749(proposed)X
2099(approach)X
2449(to)X
2546(also)X
2716(deal)X
2891(with)X
3076(the)X
3212(problem)X
3534(of)X
3635(inability)X
3959(of)X
576 5168(sampling)N
926(according)X
1299(to)X
1394(underlying)X
1802(distributions.)X
2314(In)X
2413(real)X
2571(life)X
2715(the)X
2849(samples)X
3159(are)X
3292(sometimes)X
3695(\(or)X
3822(often\))X
576 5328(provided)N
921(by)X
1042(some)X
1261(mechanical)X
1696(or)X
1802(arti\256cial)X
2130(means)X
2388(or)X
2494(good-willed)X
2953(teachers,)X
3296(rather)X
3534(than)X
3719(provided)X
576 5488(according)N
951(to)X
1048(its)X
1160(underlying)X
1570(distribution.)X
2027(Naturally)X
2387(the)X
2522(simpler)X
2814(examples)X
3174(are)X
3308(provided)X
3648(\256rst.)X
3856(Con-)X
576 5648(sider)N
770(a)X
833(situation)X
1162(where)X
1400(a)X
1463(robot)X
1673(wants)X
1902(to)X
1995(learn)X
2195(but)X
2332(there)X
2532(is)X
2615(nobody)X
2903(around)X
3171(to)X
3264(provide)X
3557(it)X
3631(with)X
3812(exam-)X
576 5808(ples)N
747(according)X
1123(to)X
1221(the)X
1358(real)X
1519(distribution.)X
2000(Because)X
2322(it)X
2401(does)X
2591(not)X
2733(know)X
2956(the)X
3092(real)X
3252(distribution,)X
3710(the)X
3846(robot)X
ep
%%Page: 42 42
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(42)X
2370(-)X
576 736(just)N
742(has)X
897(to)X
1004(generate)X
1340(its)X
1462(own)X
1651(examples)X
2022(according)X
2407(to)X
2514(its)X
2636(own)X
2825(\(computable\))X
3333(distribution)X
3779(and)X
3944(do)X
576 896(experiments)N
1034(to)X
1130(classify)X
1426(these)X
1634(examples.)X
2038(For)X
2187(example,)X
2535(in)X
2631(case)X
2808(of)X
2907(learning)X
3222(a)X
3287(\256nite)X
3495(state)X
3683(black)X
3900(box)X
576 1056(\(with)N
786(resetting)X
1114(mechanism)X
1540(and)X
1691(observable)X
2096(accepting/)X
2483(rejecting)X
2816(behavior\).)X
3221(So)X
3339(the)X
3472(sampling)X
3821(distri-)X
576 1216(bution)N
824(and)X
973(the)X
1103(real)X
1257(distribution)X
1687(may)X
1861(be)X
1966(quite)X
2165(different.)X
3 f
576 1536(7.2.)N
752(De\256nitions)X
576 1724(De\256nition.)N
1 f
1015(\(1\))X
1140(Let)X
2 f
1281(X)X
1 f
1358(be)X
1464(a)X
1526(set.)X
1669(A)X
2 f
1755(concept)X
1 f
2052(is)X
2134(a)X
2196(subset)X
2439(of)X
2 f
2535(X)X
1 f
2589(.)X
2656(A)X
2 f
2742(concept)X
3040(class)X
1 f
3240(is)X
3323(a)X
3386(set)X
2 f
3508(C)X
9 f
3588(\315)X
1 f
3665(2)X
2 f
8 s
1689(X)Y
1 f
11 s
3779 1724(of)N
3876(con-)X
576 1884(cepts.)N
803(An)X
2 f
934(example)X
1251(of)X
1344(a)X
1412(concept)X
1710(c)X
9 f
1756(\316)X
2 f
1819(C)X
1 f
1902(is)X
1985(a)X
2048(pair)X
2208(\()X
2 f
2237(x)X
1 f
2276(,)X
2 f
2305(b)X
1 f
2356(\))X
2408(where)X
2 f
2645(b)X
9 f
2703(=)X
1 f
2751(1)X
2818(if)X
2 f
2895(x)X
9 f
2941(\316)X
2 f
3004(c)X
1 f
3066(and)X
2 f
3216(b)X
9 f
3274(=)X
1 f
3322(0)X
3389(otherwise.)X
3776(A)X
2 f
3862(sam-)X
576 2044(ple)N
1 f
706(is)X
787(a)X
848(set)X
968(of)X
1063(examples.)X
776 2232(\(2\))N
903(Let)X
2 f
1046(c)X
9 f
1092(\316)X
2 f
1155(C)X
1 f
1239(be)X
1347(the)X
1480(target)X
1706(concept)X
2005(and)X
2 f
2157(P)X
1 f
2236(be)X
2344(a)X
2408(distribution)X
2841(on)X
2 f
2954(X)X
1 f
3008(.)X
3077(Given)X
2 f
3318(accuracy)X
3666(parameter)X
1 f
9 f
576 2392(e)N
1 f
615(,)X
664(and)X
2 f
818(con\256dence)X
1221(parameter)X
1 f
9 f
1614(d)X
1 f
1658(,)X
1707(a)X
2 f
1773(learning)X
2099(algorithm)X
2474(A)X
1 f
2555(draws)X
2791(a)X
2856(sample)X
2 f
3132(S)X
1 f
3202(of)X
3301(size)X
2 f
3464(m)X
8 s
3527 2409(A)N
1 f
11 s
3573 2392(\()N
9 f
3602(e)X
1 f
3641(,)X
9 f
3670(d)X
1 f
3714(\))X
3769(accord-)X
576 2552(ing)N
711(to)X
2 f
802(P)X
1 f
856(,)X
900(and)X
1049(produces)X
1388(a)X
2 f
1449(hypothesis)X
1843(h)X
9 f
1908(=)X
2 f
1970(h)X
8 s
2569(A)Y
1 f
11 s
2060 2552(\()N
2 f
2089(S)X
1 f
2140(\))X
2 f
9 f
2169(\316)X
2 f
2232(C)X
1 f
2291(.)X
776 2740(\(3\))N
900(We)X
1044(say)X
2 f
1183(C)X
1 f
1264(is)X
2 f
1345(learnable)X
1 f
1705(if)X
1781(for)X
1905(some)X
2 f
2113(A)X
1 f
2189(in)X
2280(above,)X
2534(for)X
2658(every)X
2 f
2875(P)X
1 f
2951(and)X
3100(every)X
2 f
3317(c)X
9 f
3370(\316)X
2 f
3433(C)X
1 f
3492(,)X
2 f
1943 2980(Pr)N
1 f
2038(\()X
2 f
2067(P)X
1 f
2135(\()X
2 f
2164(h)X
1 f
9 f
2215(D)X
2 f
2269(c)X
1 f
2315(\))X
2 f
2358(>)X
1 f
9 f
2431(e)X
1 f
2470(\))X
9 f
2521(\243)X
2591(d)X
1 f
2635(,)X
576 3220(where)N
9 f
815(D)X
1 f
894(denotes)X
1188(the)X
1321(symmetric)X
1719(difference.)X
2144(In)X
2242(this)X
2396(case)X
2573(we)X
2701(say)X
2844(that)X
2 f
3003(C)X
1 f
3088(is)X
3173(\()X
9 f
3202(e)X
1 f
3241(,)X
9 f
3270(d)X
1 f
3314(\))X
2 f
9 f
3343(-)X
2 f
3391(learnable)X
1 f
3729(,)X
3777(or)X
2 f
3876(pac-)X
576 3380(learnable)N
936(\(probably)X
1305(approximately)X
1836(correct)X
1 f
2090(\).)X
776 3568(\(4\))N
2 f
916(C)X
1 f
1013(is)X
2 f
1110(polynomially)X
1609(learnable)X
1 f
1985(if)X
2 f
2077(A)X
1 f
2169(runs)X
2358(in)X
2465(polynomial)X
2906(time)X
3103(\(and)X
3298(asks)X
3488(for)X
3629(polynomial)X
576 3728(number)N
867(of)X
962(examples\))X
1346(in)X
1437(1)X
2 f
1488(/)X
1 f
9 f
1520(d)X
1 f
1564(,)X
1608(1)X
2 f
1659(/)X
1 f
9 f
1691(e)X
1 f
1730(,)X
1774(and)X
1923(the)X
2053(length)X
2296(of)X
2391(the)X
2521(concept)X
2817(to)X
2908(be)X
3013(learned.)X
3 f
776 3916(De\256nition.)N
1 f
1204(A)X
1301(distribution)X
2 f
1743(P)X
1 f
1811(\()X
2 f
1840(x)X
1 f
1886(\))X
1949(is)X
2 f
2043(simple)X
1 f
2308(if)X
2397(it)X
2482(is)X
2576(\(multipli)X
2887(catively\))X
3231(dominated)X
3639(by)X
3762(a)X
3836(semi-)X
576 4076(computable)N
1010(distribution)X
2 f
1440(Q)X
1 f
1517(\()X
2 f
1546(x)X
1 f
1592(\).)X
1665(That)X
1849(is,)X
1952(there)X
2150(is)X
2231(a)X
2292(constant)X
2 f
2608(c)X
1 f
2669(such)X
2852(that)X
3007(for)X
3131(all)X
2 f
3242(x)X
1 f
3281(,)X
2 f
2073 4316(cQ)N
1 f
2189(\()X
2 f
2218(x)X
1 f
2264(\))X
9 f
2293(\263)X
2 f
2341(P)X
1 f
2409(\()X
2 f
2438(x)X
1 f
2484(\).)X
776 4584(The)N
942(\256rst)X
1108(question)X
1436(is)X
1524(how)X
1704(large)X
1909(the)X
2046(class)X
2246(of)X
2348(simple)X
2613(distributions)X
3085(is.)X
3196(It)X
3280(certainly)X
3619(includes)X
3943(all)X
576 4744(semi-computable)N
1210(distributions)X
1678(and)X
1831(hence)X
2062(all)X
2177(distributions)X
2645(in)X
2740(our)X
2882(statistics)X
3212(books.)X
3469(It)X
3548(can)X
3695(be)X
3803(shown)X
576 4904(that)N
739(there)X
945(is)X
1034(a)X
1103(non-semicomputable)X
1873(distribution)X
2311(which)X
2556(is)X
2646(simple,)X
2935(and)X
3093(that)X
3257(there)X
3464(is)X
3554(a)X
3624(distribution)X
576 5064(which)N
813(is)X
894(not)X
1029(simple.)X
ep
%%Page: 43 43
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(43)X
2370(-)X
3 f
576 736(7.3.)N
752(Discrete)X
1081(Sample)X
1382(Space)X
1 f
576 924(First)N
766(we)X
896(deal)X
1071(with)X
1256(discrete)X
1558(sample)X
1837(spaces.)X
2139(We)X
2290(show)X
2504(that)X
2666(if)X
2749(a)X
2817(concept)X
3120(is)X
3208(polynomially)X
3709(learnable)X
576 1084(under)N
804(this)X
2 f
960(single)X
1 f
1199(distribution)X
1635(then)X
1815(it)X
1893(is)X
1980(polynomially)X
2480(learnable)X
2831(in)X
2928(Valiant's)X
3279(sense)X
3497(under)X
2 f
3725(all)X
1 f
3846(`sim-)X
576 1244(ple')N
748(distributions)X
1225(if)X
1314(we)X
1451(sample)X
1736(according)X
2118(to)X
2222(the)X
2365(`universal')X
2781(distribution.)X
3246(We)X
3403(also)X
3581(provide)X
3886(new)X
576 1404(non-trivial)N
978(learning)X
1296(algorithms)X
1702(for)X
1832(several)X
2109(\(old)X
2279(and)X
2434(new\))X
2637(classes)X
2909(of)X
3010(problems)X
3366(under)X
3594(our)X
3739(assump-)X
576 1564(tion.)N
767(These)X
1008(classes)X
1283(were)X
1484(not)X
1628(known)X
1898(to)X
1999(be)X
2114(polynomially)X
2618(learnable)X
2973(under)X
3205(Valiant's)X
3560(more)X
3773(general)X
576 1724(assumption,)N
1040(some)X
1266(were)X
1476(even)X
1682(NP-complete.)X
2231(For)X
2393(example,)X
2753(the)X
2900(class)X
3110(of)X
3222(DNF's)X
3499(such)X
3699(that)X
3871(each)X
576 1884(monomial)N
966(has)X
1114(Kolmogorov)X
1595(complexity)X
2 f
2024(O)X
1 f
2101(\(log)X
2 f
2243(n)X
1 f
2294(\),)X
2376(the)X
2515(class)X
2718(of)X
2 f
2823(k)X
1 f
2862(-reversible)X
3270(DFA)X
3477(of)X
3582(Kolmogorov)X
576 2044(complexity)N
2 f
1000(O)X
1 f
1077(\(log)X
2 f
1233(n)X
1 f
1284(\),)X
1361(and)X
1513(the)X
1646(class)X
1842(of)X
2 f
1940(k)X
1 f
1979(-term)X
2195(DNF)X
2395(are)X
2527(polynomially)X
3024(learnable)X
3372(under)X
3597(our)X
3739(assump-)X
576 2204(tions.)N
822(All)X
965(these)X
1176(results)X
1436(hold)X
1623(for)X
1755(the)X
1893(appropriate)X
2324(polynomial)X
2757(time)X
2946(computable)X
3389(variants)X
3699(-)X
3759(perhaps)X
576 2364(bringing)N
897(the)X
1027(approach)X
1371(in)X
1462(the)X
1592(practicable)X
2001(domain.)X
3 f
776 2552(De\256nition.)N
1 f
1217(The)X
1379(learning)X
1693(algorithm)X
2 f
2062(samples)X
2371(according)X
2754(to)X
3 f
2849(m)X
1 f
2922(\()X
2 f
2951(x)X
1 f
2997(\),)X
3074(if)X
3154(in)X
3249(the)X
3383(learning)X
3698(phase)X
3924(the)X
576 2712(algorithm)N
957(draws)X
1203(random)X
1509(samples)X
1830(from)X
3 f
2037(m)X
1 f
2110(\()X
2 f
2139(x)X
1 f
2185(\).)X
2294(\(We)X
2481(can)X
2639(formalize)X
3013(this)X
3177(in)X
3282(different)X
3621(ways.\))X
3910(We)X
576 2872(obtain)N
819(the)X
949(following)X
1314(completeness)X
1811(result.)X
3 f
776 3060(Theorem)N
1147(6.)X
2 f
1264(A)X
1347(concept)X
1650(class)X
1855(C)X
1943(is)X
2031(polynomially)X
2521(learnable)X
2888(under)X
3123(the)X
3261(universal)X
3619(distribution)X
3 f
576 3220(m)N
2 f
649(,)X
701(iff)X
805(it)X
884(is)X
972(polynomially)X
1462(learnable)X
1829(for)X
1961(each)X
2156(simple)X
2415(distribution)X
2857(P,)X
2962(provided)X
3304(the)X
3441(sample)X
3719(is)X
3807(drawn)X
576 3380(according)N
955(to)X
3 f
1046(m)X
2 f
1119(.)X
3 f
776 3568(Proof.)N
2 f
1067(P)X
1 f
1135(\()X
2 f
1164(x)X
1 f
1210(\))X
1276(is)X
1373(dominated)X
1784(by)X
1910(some)X
2134(semi-computable)X
2780(distribution)X
2 f
3226(Q)X
1 f
3303(\()X
2 f
3332(x)X
1 f
3378(\).)X
2 f
3467(Q)X
1 f
3544(\()X
2 f
3573(x)X
1 f
3619(\))X
3686(is)X
3783(in)X
3890(turn)X
576 3728(dominated)N
971(by)X
3 f
1103(m)X
1 f
1176(\()X
2 f
1205(x)X
1 f
1251(\).)X
1346(Hence,)X
1614(there)X
1812(is)X
1893(a)X
1954(constant)X
2 f
2270(c)X
2330(>)X
1 f
2403(0)X
2469(such)X
2652(that)X
2807(for)X
2931(all)X
2 f
3042(x)X
1 f
3081(,)X
2 f
2060 3968(c)N
3 f
2106(m)X
1 f
2179(\()X
2 f
2208(x)X
1 f
2254(\))X
9 f
2305(\263)X
2 f
2375(P)X
1 f
2443(\()X
2 f
2472(x)X
1 f
2518(\))X
776 4236(Assume)N
2 f
1088(C)X
1 f
1177(is)X
1266(learnable)X
1619(\(in)X
1747(time)X
2 f
1935(t)X
1 f
1960(\))X
2019(under)X
2249(distribution)X
3 f
2709(m)X
1 f
2782(\()X
2 f
2811(x)X
1 f
2857(\).)X
2938(Then)X
3149(one)X
3306(can)X
3458(run)X
3605(the)X
3743(learning)X
576 4396(algorithms)N
990(with)X
1183(error)X
1389(parameter)X
9 f
1777(e)X
2 f
1823(/)X
1855(c)X
1 f
1930(in)X
2035(polynomial)X
2474(time.)X
2690(Let)X
2 f
2843(err)X
1 f
2985(be)X
3103(the)X
3246(set)X
3379(of)X
3487(strings)X
3757(that)X
3925(are)X
576 4556(misclassi\256ed)N
1054(by)X
1164(the)X
1294(learned)X
1575(concept.)X
1893(So)X
2008(with)X
2187(probability)X
2597(at)X
2683(least)X
2867(1)X
2 f
9 f
(-)S
1 f
9 f
2959(d)X
2 f
8 s
2022 4886(x)N
9 f
2055(\316)X
2 f
2101(err)X
1 f
16 s
9 f
2062 4822(S)N
3 f
11 s
2178 4796(m)N
1 f
2251(\()X
2 f
2280(x)X
1 f
2326(\))X
9 f
2377(\243)X
2447(e)X
2 f
2493(/)X
2525(c.)X
1 f
576 5108(Hence)N
2 f
8 s
1869 5412(x)N
9 f
1907(\316)X
2 f
1953(err)X
1 f
11 s
9 f
1924 5348(S)N
2 f
2031(P)X
1 f
2099(\()X
2 f
2128(x)X
1 f
2174(\))X
9 f
2203(\243)X
2 f
2251(c)X
8 s
2290 5412(x)N
9 f
2328(\316)X
2 f
2374(err)X
1 f
11 s
9 f
2345 5348(S)N
3 f
2452(m)X
1 f
2525(\()X
2 f
2554(x)X
1 f
2600(\))X
2 f
9 f
2629(\243)X
1 f
9 f
2677(e)X
2 f
2716(.)X
1 f
576 5634(Hence)N
832(if)X
918(the)X
1059(underlying)X
1474(distribution)X
1915(is)X
2 f
2007(P)X
1 f
2075(\()X
2 f
2104(x)X
1 f
2150(\))X
2212(rather)X
2450(than)X
3 f
2657(m)X
1 f
2730(\()X
2 f
2759(x)X
1 f
2805(\),)X
2889(we)X
3024(are)X
3164(still)X
3331(guaranteed)X
3750(to)X
3852(`pac-)X
576 5794(learn')N
2 f
805(C)X
1 f
888(\(in)X
1010(time)X
2 f
1192(t)X
1 f
1217(\),)X
1292(if)X
1370(sampling)X
1718(according)X
2088(to)X
3 f
2202(m)X
1 f
2275(\()X
2 f
2304(x)X
1 f
2350(\).)X
2446(This)X
2626(still)X
2783(requires)X
3089(that)X
3245(the)X
3376(learning)X
3688(algorithm)X
ep
%%Page: 44 44
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(44)X
2370(-)X
576 736(has)N
720(the)X
855(required)X
1176(constant)X
2 f
1498(c)X
1 f
1565(as)X
1666(additional)X
2048(input.)X
2280(The)X
2445(argument)X
2806(can)X
2956(be)X
3067(improved)X
3433(so)X
3539(that)X
3700(this)X
3856(extra)X
576 896(input)N
780(can)X
924(be)X
1029(dispensed)X
1398(with)X
1577([)X
1628(Li)X
1729(SIAM)X
1970(Learning)X
2310(simple)X
10 f
2568(`)X
1 f
776 1084(Since)N
3 f
1022(m)X
1 f
1145(assigns)X
1427(higher)X
1680(probabilities)X
2150(to)X
2247(simpler)X
2540(strings,)X
2825(one)X
2981(could)X
3206(suspect)X
3494(that)X
3656(after)X
3846(poly-)X
576 1244(nomially)N
921(many)X
1147(examples,)X
2 f
1532(all)X
1 f
1656(simple)X
1922(strings)X
2187(are)X
2324(sampled)X
2648(and)X
2804(the)X
2941(strings)X
3205(that)X
3367(are)X
3503(left)X
3650(unsampled)X
576 1404(have)N
768(only)X
951(very)X
1133(low)X
1292(\(inverse)X
1602(polynomial\))X
2061(probability.)X
2498(However,)X
2868(the)X
3003(next)X
3182(theorem)X
3498(shows)X
3744(that)X
3904(this)X
576 1564(is)N
657(not)X
792(the)X
922(case.)X
3 f
776 1752(Theorem)N
1140(7.)X
2 f
1228(Let)X
1363(S)X
1429(be)X
1534(a)X
1600(set)X
1720(of)X
1811(n)X
8 s
1865 1717(c)N
11 s
1922 1752(samples)N
2227(drawn)X
2474(according)X
2853(to)X
3 f
2966(m)X
2 f
3061(.)X
3105(Then)X
8 s
1888 2117(x)N
9 f
1921(\317)X
2 f
1967(S)X
16 s
9 f
1905 2053(S)N
3 f
11 s
1998 2027(m)N
1 f
2071(\()X
2 f
2100(x)X
1 f
2146(\))X
2 f
9 f
2197(=)X
2267(W)X
1 f
2335(\()X
2386 2099(\(log)N
2 f
2528(n)X
1 f
2579(\))X
8 s
2608 2064(2)N
11 s
2495 1975(1)N
2 f
10 f
2373 2001(h)N
2397(hhhhhh)X
1 f
2669 2027(\).)N
776 2367(Now)N
979(let)X
1102(us)X
1214(consider)X
1546(polynomially)X
2052(computable)X
2498(distributions.)X
2996(Again,)X
3267(all)X
3390(textbook)X
3733(distribu-)X
576 2527(tions)N
777(we)X
908(know)X
1132(are)X
1268(polynomially)X
1769(computable.)X
2232(Call)X
2409(a)X
2477(distribution)X
2 f
2914(polynomial)X
3340(simple)X
1 f
3599(if)X
3681(it)X
3759(is)X
3846(dom-)X
576 2687(inated)N
816(by)X
928(a)X
991(polynomially)X
1487(computable)X
1923(distribution.)X
2399(In)X
2496(all)X
2609(of)X
2706(the)X
2838(discussion)X
3229(below)X
3468(all)X
3582(Kolmogorov)X
576 2847(complexity)N
996(\(including)X
1381(the)X
1511(related)X
1773(notion)X
3 f
2021(m)X
1 f
2094(\))X
2145(can)X
2289(be)X
2394(replaced)X
2714(by)X
2824(its)X
2930(polynomial)X
3355(bounded)X
3680(version.)X
3 f
576 3167(7.3.1.)N
818(Learning)X
1188(under)X
1435(m\(x\))X
1 f
576 3355(In)N
675(the)X
809([)X
864(Li)X
969(1989)X
1171(Simple)X
1448(Concepts)X
1802(we)X
1930(gave)X
2122(an)X
2231(example)X
2556(of)X
2655(a)X
2720(class)X
2917(of)X
3016(simple)X
3278(concepts,)X
3635(log)X
2 f
3762(n)X
1 f
(-DNF,)S
576 3515(which)N
815(is)X
898(polynomially)X
1394(learnable)X
1741(under)X
1965(the)X
2097(universal)X
2444(distribution,)X
2897(and)X
3047(hence)X
3275(in)X
3367(our)X
3507(sense)X
3720(under)X
3943(all)X
576 3675(simple)N
837(distributions,)X
1326(and)X
1478(which)X
1718(is)X
1803(not)X
1942(known)X
2207(to)X
2302(be)X
2411(polynomially)X
2909(learnable)X
3258(in)X
3353(the)X
3487(general)X
3772(Valiant)X
576 3835(model.)N
875(Here)X
1079(we)X
1215(present)X
1503(a)X
1576(class)X
1781(that)X
1948(was)X
2118(shown)X
2381(to)X
2484(be)X
2600(not)X
2746(polynomially)X
3251(learnable)X
3607(in)X
3709(Valiant's)X
576 3995(sense,)N
810(unless)X
1052(P)X
1123(=)X
1195(NP,)X
1351(but)X
1486(which)X
1723(is)X
1804(polynomially)X
2298(learnable)X
2643(under)X
3 f
2865(m)X
1 f
2938(\()X
2 f
2967(x)X
1 f
3013(\).)X
776 4183(DNF)N
978(stands)X
1225(for)X
1354(`disjunctive)X
1798(normal)X
2075(form'.)X
2346(A)X
2436(DNF)X
2638(is)X
2724(any)X
2878(sum)X
2 f
3052(m)X
1 f
8 s
3125 4200(1)N
2 f
11 s
9 f
3164 4183(+)N
2 f
3212(m)X
1 f
8 s
3285 4200(2)N
2 f
11 s
9 f
3324 4183(+)N
1 f
3372(...)X
2 f
9 f
(+)S
2 f
3486(m)X
8 s
3549 4200(r)N
1 f
11 s
3608 4183(of)N
3708(monomi-)X
576 4343(als,)N
719(where)X
955(each)X
1138(monomial)X
2 f
1519(m)X
8 s
1582 4360(i)N
1 f
11 s
1651 4343(is)N
1732(the)X
1862(product)X
2153(of)X
2248(some)X
2456(literals)X
2719(chosen)X
2985(from)X
3178(a)X
3239(universe)X
2 f
3559(x)X
1 f
8 s
3608 4360(1)N
11 s
3647 4343(,)N
3691(.)X
3735(.)X
3779(.)X
3823(,)X
2 f
3859(x)X
8 s
3898 4360(n)N
1 f
11 s
3959 4343(or)N
576 4507(their)N
760(negations)X
2 f
1120(x)X
1 f
10 f
1120 4448(h)N
8 s
1 f
1169 4524(1)N
11 s
1208 4507(,)N
1252(.)X
1296(.)X
1340(.)X
1384(,)X
2 f
1420(x)X
1 f
10 f
1420 4448(h)N
2 f
8 s
1459 4524(n)N
1 f
11 s
1498 4507(.)N
1564(A)X
2 f
1649(k)X
1 f
1688(-term)X
1901(DNF)X
2098(is)X
2179(a)X
2240(DNF)X
2437(consisting)X
2817(of)X
2912(at)X
2999(most)X
2 f
3194(k)X
1 f
3256(monomials.)X
3716(A)X
3802(mono-)X
576 4667(mial)N
756(in)X
847(a)X
908(DNF)X
1105(is)X
2 f
1186(monotone)X
1 f
1555(if)X
1631(no)X
1741(variable)X
2047(in)X
2138(it)X
2210(is)X
2291(negated.)X
2631(In)X
2726([)X
2777(Pitt)X
2923(Valiant)X
3205(35)X
3315(1989)X
598 4827(it)N
671(was)X
830(shown)X
1082(that)X
1238(learning)X
1551(a)X
1614(monotone)X
2 f
1991(k)X
1 f
2030(-term)X
2245(DNF)X
2444(by)X
2 f
2556(k)X
1 f
2595(-term)X
2810(\(or)X
2936(2)X
2 f
(k)S
1 f
3019(-term\))X
3263(DNF)X
3462(is)X
3545(NP-complete.)X
576 4987(\(In)N
706(contrast)X
1013(to)X
2 f
1110(k)X
1 f
1149(-DNF)X
1381(is)X
1467(a)X
1533(DNF)X
1735(where)X
1976(each)X
2164(monomial)X
2550(consists)X
2856(of)X
2956(at)X
3047(most)X
2 f
3246(k)X
1 f
3312(literals.)X
3624(Recall,)X
3899(that)X
2 f
576 5147(k)N
1 f
615(-DNF)X
841(is)X
922(learnable)X
1267(in)X
1358(Valiant's)X
1703(sense)X
1915([)X
1966(Valiant)X
2248(1984)X
2446(learnable)X
2791(\))X
3 f
776 5335(Theorem)N
1153(8.)X
2 f
1254(Monotone)X
1646(k-term)X
1911(DNF)X
2123(is)X
2218(polynomially)X
2715(learnable)X
3089(by)X
3208(monotone)X
3591(k-term)X
3856(DNF)X
576 5495(while)N
790(sampling)X
1135(under)X
3 f
1362(m)X
2 f
1435(.)X
776 5683(Proof)N
1104(\(Sketch\).)X
1 f
1563(Assume)X
1973(we)X
2202(are)X
2436(learning)X
2852(a)X
3018(monotone)X
2 f
3499(k)X
1 f
3538(-term)X
3857(DNF)X
2 f
583 5843(f)N
1 f
636(\()X
2 f
665(x)X
1 f
8 s
714 5860(1)N
11 s
753 5843(,)N
804 5817(.)N
848(.)X
892(.)X
936 5843(,)N
2 f
965(x)X
8 s
1004 5860(n)N
1 f
11 s
1043 5843(\))N
2 f
9 f
1086(=)X
2 f
1148(m)X
1 f
8 s
1221 5860(1)N
2 f
11 s
9 f
1260 5843(+)N
1 f
1330 5817(.)N
1374(.)X
1418(.)X
2 f
9 f
1462 5843(+)N
2 f
1510(m)X
8 s
1573 5860(k)N
1 f
11 s
1608 5843(,)N
1652(where)X
2 f
1888(m)X
8 s
1951 5860(i)N
1 f
11 s
1976 5843('s)N
2061(are)X
2190(the)X
2 f
2320(k)X
1 f
2381(monotone)X
2756(monomials)X
3171(\(terms\))X
3447(of)X
2 f
3549(f)X
1 f
3574(.)X
ep
%%Page: 45 45
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(45)X
2370(-)X
3 f
576 736(Learning)N
946(Algorithm.)X
1 f
576 924(0.)N
776(Draw)X
993(a)X
1055(sample)X
1328(of)X
2 f
1424(n)X
8 s
1478 889(k)N
9 f
1506(\242)X
1 f
11 s
1552 924(examples,)N
2 f
1930(k)X
9 f
1969(\242)X
2 f
1991(>k)X
9 f
2103(+)X
1 f
2151(1.)X
2240(Set)X
2376(DNF)X
2 f
2574(g)X
1 f
2639(:)X
2 f
9 f
2664(=)X
2726(\306)X
1 f
2799(.)X
2866(\()X
2 f
2895(g)X
1 f
2962(is)X
3045(the)X
3177(DNF)X
3376(we)X
3502(will)X
3664(eventually)X
776 1084(output)N
1024(as)X
1119(approximation)X
1656(of)X
2 f
1758(f)X
1 f
1783(.\))X
576 1272(1.)N
776(Pick)X
955(a)X
1016(positive)X
1318(example)X
2 f
1639(a)X
9 f
1697(=)X
1 f
1745(\()X
2 f
1774(a)X
1 f
8 s
1828 1289(1)N
11 s
1867 1272(,)N
1918 1246(.)N
1962(.)X
2006(.)X
2050 1272(,)N
2 f
2079(a)X
8 s
1289(n)Y
1 f
11 s
2162 1272(\).)N
2235(Form)X
2448(a)X
2509(monotone)X
2884(term)X
2 f
3068(m)X
1 f
3153(such)X
3336(that)X
2 f
3491(m)X
1 f
3576(includes)X
2 f
3892(x)X
8 s
3931 1289(i)N
1 f
11 s
3978 1272(if)N
2 f
776 1432(a)N
8 s
1449(i)Y
11 s
9 f
845 1432(=)N
1 f
893(1.)X
576 1620(2.)N
2 f
776(for)X
1 f
914(each)X
1110(positive)X
1425(example)X
2 f
1759(a)X
9 f
1824(=)X
1 f
1886(\()X
2 f
1915(a)X
1 f
8 s
1969 1637(1)N
11 s
2008 1620(,)N
2 f
2037(...)X
1 f
(,)S
2 f
2132(a)X
8 s
1637(n)Y
1 f
11 s
2215 1620(\))N
2 f
2279(do)X
1 f
(:)S
2428(if)X
2 f
2518(a)X
8 s
1637(i)Y
11 s
9 f
2601 1620(=)N
1 f
2663(0)X
2743(and)X
2906(deleting)X
2 f
3227(x)X
8 s
3266 1637(i)N
1 f
11 s
3327 1620(from)N
2 f
3534(m)X
1 f
3633(violates)X
3944(no)X
776 1780(negative)N
1097(examples,)X
1474(delete)X
2 f
1707(x)X
8 s
1746 1797(i)N
1 f
11 s
1793 1780(from)N
2 f
1986(m)X
1 f
2049(.)X
576 1968(3.)N
776(Remove)X
1100(from)X
1301(the)X
1439(sample)X
1719(all)X
1838(positive)X
2148(examples)X
2511(which)X
2756(are)X
2893(implied)X
3194(by)X
2 f
3312(m)X
1 f
3375(.)X
3449(Set)X
2 f
3593(g)X
1 f
9 f
3658 MX
(<-)172 987 oc
2 f
3759(g)X
9 f
3810(+)X
2 f
3858(m)X
1 f
3921(.)X
3974(If)X
776 2128(there)N
974(are)X
1103(still)X
1259(positive)X
1561(examples)X
1916(left,)X
2078(then)X
2252(go)X
2362(to)X
2453(step)X
2617(1,)X
2705(else)X
2864(halt)X
3019(and)X
3168(return)X
2 f
3400(g)X
1 f
(.)S
776 2476(We)N
922(show)X
1131(that)X
1288(the)X
1420(algorithm)X
1788(is)X
1871(correct.)X
2183(Let)X
2325(us)X
2427(write)X
2 f
2632(m)X
8 s
2695 2493(i)N
11 s
9 f
2734 2476(\315)N
2 f
2811(m)X
1 f
2898(for)X
3025(two)X
3182(monotone)X
3560(monomials)X
3978(if)X
576 2636(all)N
689(the)X
821(variables)X
1163(that)X
1320(appear)X
1578(in)X
2 f
1671(m)X
8 s
1734 2653(i)N
1 f
11 s
1783 2636(also)N
1949(appear)X
2207(in)X
2 f
2300(m)X
1 f
2363(.)X
2431(At)X
2543(step)X
2709(1,)X
2799(the)X
2931(monomial)X
2 f
3314(m)X
1 f
3400(obviously)X
3771(implies)X
576 2796(no)N
687(negative)X
1009(examples,)X
1387(since)X
1591(for)X
1716(some)X
1925(monomial)X
2 f
2308(m)X
8 s
2371 2813(i)N
1 f
11 s
2420 2796(of)N
2 f
2524(f)X
1 f
2573(we)X
2699(must)X
2895(have)X
2 f
3085(m)X
8 s
3148 2813(i)N
11 s
9 f
3187 2796(\315)N
2 f
3264(m)X
1 f
3327(.)X
3373(Step)X
3554(2)X
3622(of)X
3719(the)X
3851(algo-)X
576 2956(rithm)N
797(keeps)X
1026(deleting)X
1340(variables)X
1687(from)X
2 f
1887(m)X
1 f
1950(.)X
2001(If)X
2088(at)X
2181(any)X
2337(time)X
2524(for)X
2655(no)X
2771(monomial)X
2 f
3158(m)X
8 s
3221 2973(i)N
11 s
9 f
3246 2956(\316)N
2 f
3316(f)X
1 f
3369(holds)X
2 f
3588(m)X
8 s
3651 2973(i)N
11 s
9 f
3690 2956(\315)N
2 f
3767(m)X
1 f
3830(,)X
3880(then)X
576 3116(there)N
778(exists)X
1005(a)X
1070(negative)X
1395(example)X
1720(that)X
1879(contains)X
2199(at)X
2289(most)X
2 f
2487(k)X
1 f
2553(0's)X
2687(such)X
2875(that)X
3035(it)X
3112(satis\256es)X
2 f
3418(m)X
1 f
3508(but)X
3648(no)X
2 f
3763(m)X
8 s
3826 3133(i)N
1 f
11 s
3878 3116(of)N
2 f
3985(f)X
1 f
4010(.)X
576 3276(This)N
762(negative)X
1090(example)X
1418(is)X
1506(of)X
1608(Kolmogorov)X
2086(complexity)X
2512(at)X
2604(most)X
2 f
2804(k)X
1 f
2850(log)X
2 f
2977(n)X
1 f
(,)S
3071(hence)X
3304(by)X
3420(the)X
3556(Chernoff)X
3901(for-)X
576 3436(mulae)N
825(\(Section)X
1152(2.1\))X
1324(with)X
1514(high)X
1704(probability)X
2125(it)X
2208(is)X
2300(contained)X
2676(in)X
2778(the)X
2919(sample.)X
3246(Hence)X
3503(at)X
3600(step)X
3775(2,)X
3875(with)X
576 3596(high)N
767(probability,)X
1211(there)X
1421(will)X
1593(be)X
1710(an)X
2 f
1827(m)X
8 s
1890 3613(i)N
1 f
11 s
1949 3596(such)N
2144(that)X
2 f
2311(m)X
8 s
2374 3613(i)N
11 s
9 f
2413 3596(\315)N
2 f
2490(m)X
1 f
2553(.)X
2631(Hence)X
2889(we)X
3024(eventually)X
3425(\256nd)X
3595(a)X
3667(correct)X
2 f
3944(m)X
8 s
4007 3613(i)N
1 f
11 s
576 3756(\(precisely\))N
984(with)X
1173(high)X
1362(probability.)X
1804(Then)X
2017(at)X
2114(step)X
2289(3,)X
2388(we)X
2523(remove)X
2820(the)X
2961(positive)X
3274(examples)X
3640(implied)X
3944(by)X
576 3916(this)N
2 f
730(m)X
8 s
793 3933(i)N
1 f
11 s
844 3916(and)N
997(continue)X
1327(on)X
1441(to)X
1536(\256nd)X
1699(another)X
1988(term)X
2175(of)X
2 f
2280(f)X
1 f
2305(.)X
2374(The)X
2536(algorithm)X
2905(will)X
3068(eventually)X
3461(output)X
2 f
3712(g)X
9 f
3770(=)X
2 f
3825(f)X
1 f
3875(with)X
576 4076(high)N
755(probability)X
1165(by)X
1275(standard)X
1595(calculations.)X
10 f
2061(`)X
2 f
776 4264(Remark.)N
1 f
1094(Notice)X
1352(that)X
1508(this)X
1659(is)X
1741(not)X
1878(an)X
1985(approximation)X
2524(algorithm)X
2892(like)X
3049(the)X
3181(one)X
3332(in)X
3425([)X
3478(Li)X
3581(1989)X
3781(Simple)X
576 4424(Concepts)N
598 4584(to)N
689(learn)X
887(log)X
2 f
1014(n)X
1 f
(-DNF.)S
1306(This)X
1485(algorithm)X
1851(outputs)X
2133(the)X
2263(precise)X
2534(monotone)X
2909(formula)X
3210(with)X
3389(high)X
3568(probability.)X
3 f
576 4904(7.4.)N
752(Continous)X
1160(Sample)X
1461(Space)X
1 f
576 5092(Secondly,)N
951(we)X
1079(deal)X
1252(with)X
1435(continuous)X
1848(sample)X
2124(spaces.)X
2423(For)X
2571(example,)X
2918(the)X
3052(uniform)X
3362(distribution)X
3796(now)X
3973(is)X
576 5252(de\256ned)N
865(as)X
2 f
968(L)X
1 f
1031(\()X
9 f
1060(G)X
2 f
8 s
1113 5269(x)N
1 f
11 s
1148 5252(\))N
2 f
9 f
1191(=)X
1 f
1253(2)X
2 f
8 s
9 f
5217(-)Y
1 f
9 f
1342(|)X
2 f
1365(x)X
1 f
9 f
1413(|)X
11 s
1 f
1443 5252(,)N
1495(where)X
9 f
1739(G)X
2 f
8 s
1792 5269(x)N
1 f
11 s
1856 5252(denotes)N
2154(the)X
2291(set)X
2418(of)X
2520(all)X
2638(one-way)X
2969(in\256nite)X
3249(binary)X
3503(strings)X
3767(starting)X
576 5412(with)N
2 f
758(x)X
1 f
797(.)X
844(This)X
1026(is)X
1111(the)X
1245(Lebesgue)X
1608(measure)X
1927(on)X
2041(interval)X
2337([0,)X
2439(1].)X
2560(While)X
2802(for)X
2930(discrete)X
3230(sample)X
3506(spaces)X
3761(all)X
3876(con-)X
576 5572(cept)N
757(classes)X
1035(are)X
1176(Valiant)X
1470(learnable)X
1827(\(although)X
2199(not)X
2346(all)X
2469(are)X
2610(polynomially)X
3116(learnable\),)X
3524(this)X
3686(is)X
3778(not)X
3924(the)X
576 5732(case)N
749(for)X
873(continuous)X
1282(sample)X
1554(spaces.)X
1849(We)X
1994(can)X
2139(de\256ne)X
2377(the)X
2508(notion)X
2757(of)X
2853(`simple')X
3170(semimeasure)X
3653(and)X
3803(that)X
3959(of)X
576 5892(universal)N
944(semicomputable)X
1568(semimeasure,)X
2095(over)X
2296(a)X
2380(continuous)X
2811(sample)X
3105(space,)X
3366(and)X
3537(show)X
3766(that)X
3943(all)X
ep
%%Page: 46 46
bp
11 s 0 xH 0 xS 1 f
10 s
0 32(--)N
4323(--)X
11 s
2209 424(-)N
2260(46)X
2370(-)X
576 736(concept)N
882(classes)X
1158(are)X
1297(learnable)X
1653(over)X
1842(each)X
2036(simple)X
2305(semimeasure)X
2 f
2798(D)X
1 f
2894(iff)X
3010(they)X
3195(are)X
3335(learnable)X
3691(under)X
3924(the)X
576 896(universal)N
928(semimeasure.)X
1439(In)X
1541(contrast)X
1849(with)X
2035(the)X
2172(discrete)X
2475(case)X
2654(w.r.t.)X
2865(polynomial)X
3296(learning,)X
3635(here)X
3814(we)X
3944(do)X
576 1056(not)N
717(need)X
911(to)X
1008(require)X
1285(that)X
1446(the)X
1582(learning)X
1899(algorithm)X
2271(samples)X
2583(according)X
2958(to)X
3055(the)X
3192(universal)X
3544(measure.)X
3910(For)X
576 1216(details,)N
851(see)X
985([)X
1036(Li)X
1137(1989)X
1335(Simple)X
1608(Concepts)X
3 f
10 s
576 1504(8.)N
676(Acknowledgement)X
1 f
576 1676(We)N
712(are)X
835(grateful)X
1109(Ray)X
1262(Solomonoff,)X
1688(Leonid)X
1939(Levin,)X
2170(and)X
2310(Peter)X
2499(G)X
2571(\302)X
2557(acs)X
2685(for)X
2804(conversations)X
3270(about)X
3473(algorithmic)X
3867(com-)X
576 1820(plexity)N
824(and)X
966(inductive)X
1290(reasoning.)X
1648(We)X
1786(thank)X
1990(Mati)X
2167(Wax)X
2344(for)X
2463(useful)X
2684(discussions)X
3073(on)X
3178(the)X
3301(MDL)X
3504(principle;)X
3836(Leslie)X
576 1964(Valiant)N
832(for)X
946(suggesting)X
1308(handwritten)X
1711(character)X
2027(recognition)X
2412(as)X
2499(a)X
2556(viable)X
2773(application;)X
3172(and)X
3309(Qiong)X
3530(Gao)X
3685(for)X
3800(partici-)X
576 2108(pating)N
803(in)X
892(the)X
1017(realization)X
1383(of)X
1477(it.)X
1568(Fahiem)X
1835(Bacchus,)X
2154(Danny)X
2395(Krizanc)X
2677(and)X
2820(John)X
2998(Tromp)X
3243(read)X
3409(and)X
3552(commented)X
3952(on)X
576 2252(the)N
694(manuscript.)X
0 6360(--)N
4323(--)X
ep
%%Trailer
xt
xs
%%Pages: 46
%%DocumentFonts: DIThacks
%%+ Symbol
%%+ Times-Bold
%%+ Times-Italic
%%+ Times-Roman
%%EOF