From 2c206e9ce525ae63d1db8756db0d8d019f703d49 Mon Sep 17 00:00:00 2001 From: XianBW <36835909+XianBW@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:00:56 +0800 Subject: [PATCH] docs: some changes in kaggle doc (#461) * add kaggle scenario description for webapp * catch download button error * change kaggle auto_submit to False default * Added guidance for custom templates in kaggle doc * CI * CI * add kaggle data set instruction(NOTE) * delete repeated description for kaggle scenario --------- Co-authored-by: TPLin22 --- docs/scens/kaggle_agent.rst | 26 ++++++++++++++++++ docs/scens/kaggle_template.png | Bin 0 -> 12655 bytes rdagent/app/kaggle/conf.py | 2 +- rdagent/log/ui/app.py | 16 +++++++---- .../scenarios/kaggle/experiment/scenario.py | 24 +++++++++++++++- rdagent/scenarios/kaggle/kaggle_crawler.py | 10 ++++--- rdagent/utils/env.py | 2 -- 7 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 docs/scens/kaggle_template.png diff --git a/docs/scens/kaggle_agent.rst b/docs/scens/kaggle_agent.rst index 272b2055..bea1e676 100644 --- a/docs/scens/kaggle_agent.rst +++ b/docs/scens/kaggle_agent.rst @@ -100,6 +100,10 @@ You can try our demo by running the following command: The `competition name` parameter must match the name used with the API on the Kaggle platform. +(NOTE: The code for crawling Kaggle competition information may not be applicable to your environment. + +If you cannot execute it normally, you can refer to the following **Example Guide: Running a Specific Experiment**.) + 📋 Competition List Available ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -149,6 +153,8 @@ You can try our demo by running the following command: - Alternatively, you can manually place the dataset in the specified location in advance. + - you can download the kaggle_data.zip in the release and unzip it to the directory configured by the `KG_LOCAL_DATA_PATH` environment variable. + - 🚀 **Run the Application** @@ -166,6 +172,26 @@ You can try our demo by running the following command: For more information about Kaggle API Settings, refer to the `Kaggle API `_. + +🎨 Customize one template for a new competition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In order to facilitate RD-Agent to generate competition codes, we have specified a competition code structure: + +.. image:: kaggle_template.png + :alt: Design of Kaggle Code Template + :align: center + +- **feature directory** contains the feature engineering code. Generally no modification is required. +- **model directory** contains the model codes. + select_xx.py is used to select different features according to different models. + model_xx.py is the basic code of different models. Generally, only some initial parameters need to be adjusted. +- **fea_share_preprocess.py** is some basic preprocessing code shared by different models. The degree of customization here is high, but the preprocess_script() function needs to be retained, which will be called by train.py +- **train.py** is the main code, which connects all the codes and is also the code called during the final execution. + +**We will soon provide a tool for automatic/semi-automatic template generation.** +If you want to try a different competition now, you can refer to our current template structure and content to write a new template. + + 🎯 Roadmap ~~~~~~~~~~~ diff --git a/docs/scens/kaggle_template.png b/docs/scens/kaggle_template.png new file mode 100644 index 0000000000000000000000000000000000000000..022c892096140185171cf5eb5191cfe4f027f935 GIT binary patch literal 12655 zcma*OWmH_*@ zZLgglt5wyHS~b_4WAxE`AESchWW-Pq@ex5F5Q>EOX9eK(1_XkDhKB{-T|Q;Y0WS~^ z3SvT_ve9>Yzz1j(L1{q{s4^1iQ4a?Aj9@FS?f?RzcD-H@eKrM#AP_UO#AiVzSM8&8 z4_ks6uah>cD>5=kHfli`%6K_?G6Qlm-FZ7lINz@vP6|jeDIX&+qq^Cuw0fpCBCN{` zRF~My97K4HS<)@!r5&lwj!(aJv;LY$wqUB4_DrUQ59g`GWn-tntD$c%(}#OYtr3?6 z=il>Wpkp$|@D>CUGqiQW+7(@)md4u@y7D~A75bD`lt<-41T8ol;r0mWOL7PS!znl$ zSF~rH;Cn$Xw{0OMkRLAHx)Hu~uDI zCXTqRm6Qo4uj9aqjfa)allLBH_52@Ep7iWje-px?n>YIqAKsIWp4mXDZD~P*Zf(5p zK4{<1i99Z(;(@UVi_>dw%|i*AbRNAQHIc1$k?VCWNf2&C%UvPxBR6&vx`G57vu#Sd zbZzsWbC;P5QM(DmM|(X5(`)|?Vz^gdn5eN0j>$_jyAjQ%%zsq zAQo^gKuV45j|LV|cd=Z*~e7S6C_(lN!)jt;@$9;LfKvT?B6du?1uwHMUD zPc?M);D9F^pvmNX$n`En=+iQUc&)IOdRZxfjt#y4kQ_Xrz-+ACzVe_+cO!v~sskU= z4=yT@0+d3ihy$Nuka(Xa9c!>_3(m5}-2+xvz{Wz-%;HTHSN;74!%X^;>YW;-$EJPw zzMd_^h%Up@n!HX&u~7V1xp)8 zDBk@WGY{9;?W^h(DmDjezpp_v7LiRt{auH}kAy*M;|VIC|L9l1Z6UcwJE+53WpaDq zudMXJH9$r3y%2iloI!#wkBHZ6_;P8Rou9qH^77NAg&R27B%!X6_acYk-O*N;xM`1k zhl*w9%xQQDv~M8S zkQ2qSh%6GuA&kDt`tI$dTDlbdPJhXORybr*U9`AvFdr?wY_Vm#bL>OreVFo! z?2UT2v#k07%z%_7n>sgf`^=}?cheei!q*0RfzxSMooP{KtANnm4PPPkaDa^HLAtF+ z{8O_{1eQ1Z%t&56b=%p_*-|CE-d0&H!9#G~5r|u{jDnmggzSSR;QSdOD`vbj6JL0` z{AC^IeZWx8!jEGK*hG}7cO=@590$wXVIvI8x{zO3Q9)V+4;ah+C7<|IG@i^`W{Hmb zoQ>rNDZ!+X%M0|aeQaVLH+B0ASS=n+75ldw-@6uSMs6~m;3akTul=6jiA_>!{p*Ss zcBp?a``CVLr|Z`rM$^(y_!2USTNQxwfp`XbjI8Xdpm3IhdchI&*X@LXjNVYjN8FQ! zg|qNCP@3DdNLxlHPO&J~FEk(f`W)eO_FN>SlX5I!BZzue3kV1v^<#P5RCw7Q<87_{ z?%GM)__UmC!?ZbgXLCD@Xn`*!Ck9*|a+=B*Vlneo9-f7g|xsYz&ZOqhUwNdtY4j0VW-ChgNB;%;cOD^w299*C%r^@S~_A#TIVp0-A6JozlPnQAiJUNtWRnkAgUa2AS)H*5q7&$V9isTZBe)LE0#9XB(h6J<9|IeaHHf`t^Y2=xNV^P=m&M>Q?R8OM=1pv$n)< zl@h^A4R#Nx&v%KQP^lt~*v7UNKRxHi@i^LQp$qv7@J>mC^l=xh(%(3O-e+LmHjK-#3-S67iKfU?E!jR!}E2jY&Hz`4%82ARdpo^F^Fk zo~>Qgmyu-R6f3JX!5eJUhnOb)xcjvh_uL&tVvNB0W--&Wq%Rw0>QVo3OSPq-tgQ2Q zBDj?9qA~q;(`zUc)`j}HBwOup#}C?$GY)$(;@+rzH;^il@l!P2K?Uk*G(`%7XAgze^--0?mJ0fD&n!f2?l3zA;^Dk&7j3FQ6%`@5-rWl9-WcyP-64@wz44Wi-a-hWc8=IBNA9$)&q-DA#lfb`~g5x}Q8+Iw{BAe;z zN8%E=A41|9)>IlQ|N5Q7Q4-AomzQ1?-MK5K<7>-L2qcxoKu>GPdWM#eg_GyhS>!P! z0lvT@Pt>bgNN4Di4n=RR6jS+{pFv};vKk(TE8&=HP^LSHnBQS4wQL#H_Ij(vPJ~Lm zSrd3J*~}Ls_3UDa*`@h5b4${3>`eirc3}2&uAH>{^_F&}b{F{3(4o`358k#m?;z^Lby`;E4Av#VLrU^)2EXlQ zq0A!xk;wAq`}Zvtm#Y|T$rXHHi^No!?Y7=fmHlM0qYMwGH8W&#N?I|z@hK}rs7D0^G zwHN?9ia8Lz=}>{LeKbn!nOZ=x%xh4TmI5h4jX&D@a5*g!Ha|nK5ai6uR>U);%};qUsmf?p`SM>nUD?(_9)O}rssA)KjLy1z1F(-T2~Tj)T9TP z4wyqrhz4$#q;QQd#{QFSZ&`0wG9S^=%8Gl(w41^+TR#Q2Ueo=E+Y6#)DW=x5piBN` zB$GL$zY~pidZAi(9tIknmn{Z5<9WUQSboP%dBt%C%HL(eyt^+v0Q&S8kZ zBHDXW2;Kk3D~#*!8FLAu#|HifkpD++S1x)TV_WbC7ZuTrR4s7wch1_ZOw*CWcfON? zxj9*Sf#-OFHm6SxO(I-|ipnmz`Ux}gY%N`fehk@su zA&oK)upSgx0`+=evuZ(K)D5K53S5VTN#_LkqhFcFH!&K zDRW!36ojWWy&J6Gz*96Y8#?kO+cduONx!-c9AD{?Mq(Uw?nj{S({|e#c^1*sve};C zmN~R6g`>Lh%>!rSSy}3Ag_Aev<-X@Z{-zS<_K%6mMzqsS`%AKH$HZh%N@P6uSteey z%xs|Ym3o*2^N7iMEsEuPiqM8d!^D?5rc#h^1$-=$wK&AHYU(LYlO&oY$n_+AhOIpE zWr(1`^$s6I@_v&!dWhWl3=y1;uQ=^?BMdRbcL%er=psz~#tg2R)Hp9v4x}Aa)iFCq zQB~qL*+xJ=*sh6e5|H&oC#V)T0Cz*N#(EQP!FhJW??#yZ;z-o;+vW87VTNS$;>~6vIN4T!8(sN6H{D(k6`atB7nB$c^ zV5@K)YqnubO(B6mzf;jb%F1fL97*{qow02Vu-ZHy@V|4rM5Ugz=VWSyKVG3R-A|X> zZmIIGg_6N%gQieWP*&D(WY|?-3~!hoQW@$vI1m9*gI?0|^f68B;hAFrO&O8Z0lAua1hT#vA>bdO;v= zx+AjIW+rZ-f)|Q30`MhhkmU%<93Ycu|A8+5=94{t>N|_J8GSbtWp&r~Wpm6s$OQTY z8@M7g3w73ddj!Ewz9hJG>A!Hw?u3<;8Qa5=%7Z$)n|<7nEaa+317U{t@XURgy8d*8 zdlFrwA5@qgb@@aoKA#JrUc9AWo5^7IBwV7LQv>{{#G2*F@qp0^^PIvD&r2UMj>SGM zmhwr94S!5PKHjH9QL9+z+dTQQI-RR(_ps!RnYspWlO4JEv8+~(NWu<}(@A2@_x(FG zM-Yrmlm5D=TvNKN@!wHNN=z$*gwz-sa7r8v=5}1NrrYqOh`{*l$9)wI1Cc3tDE(6E z+>D#!a;aSr<%~TYM>ng<{J-&+g7)5D@a@r*1Z=@hCh)dmPSw)N$K&|nx(QlF?Bp7| zX$!nEimAl$dK)4V_>V8s^qy{G*sUJ|)b+)#PI`k&oy1YO)~uX^oFthS{!A*)=YOME zB*?uOq*l`7o zW-)AeedQ%s5|E|!?O^C$>$yIq8aV*y4oLrFO)hqH4X!}8brDLsW=lVEKYkGdMQWXy z|69qqhu^eQFz5GuF}60?H_xp^8I%=SfulPhNDMOhs&fDW<3g}r<^?t36ekGQ#s&m;pn7ta=a_z2$kL0O% zz0iRRM0)_*{Auj|O0-mEIwlrye@&T5@v?O6pVsv3?mxl8k z#c#q)z`-!?`;joEV&zw_@@bm)!On4(zSRW>%Jk4~Pr3jn<*zm74pBppfOM?6=THzM z+M~)iCgo+#EG*nF^^%ydbJ5_XDR48yd8dE;rg&C97hJ1A6zdh*dF1Dfoi;4B1k=}s zIC{pcrJpGM=J~c=84k{ySx=%^+U8Dnw7HUej1<=r+wYMi_Gw7h%1@Z^^dOi-`$Vv5 zl0p`j{lLn1%lBbB^(7<{e8FN(AxrMPllP}wRHi+YzcB>I`;k=Ja}VG7xeHgwn=@eS z1iy<`IDV2*y+2}K2j+e1H({cQ;Zm|AVF8qYYoAZ;l-Iml;Ocw600V!hiqG8}IoD6| zWs+IQbFt%L*jSx4!J{kq24k|jAjlMwh_&Gg%nU!GkAY-$Rxw>69Vnps<=~B8Tjaao z1@q}qq}j7tN>i_9*p+()l(#BS<=RQ@Hsx`DRv6HRX61oGYTqC1qm=+hX>z>W0{De+ zai3h@SWcgGwdz7AZ(9vMU=uI{_RqF~-N*Q{0gg!#BxFwr{jY2X#nQGwa!j;9kezew z28;W7!bDhoLioNp#)ZM(ySNv{ATTQWKE{N$qU~+3eSN1Ikyrn><2H5DtMVS?=bQKZ z-F^o}XVo*-8~NobjpfqA&YPNghxCGkuV4r%Bh%Q##$92!oxtVA5SlK=*FL5*vhMer zTR^-0!du1fM%3PQYw-}pQd)Gq3Y+<=2CliE$%XAOB$t&_FfP<=8&!f|V;y{Gp!!!I zsx_+1pm%Tn8|VK&zCkN?UVSDay0td>1GQkBPcCb{K7Y$?$oWs)Ia!2kHafz^kf+J3 ziglS-dU|C;e&tgqM{=HlK2n4S{ee}Lqx^dIYeg;(@=RfI;nCqN zYCP<*7TECz;E#e4Ndp5kz8&U6>-qi*WiJxTlqiGo9}1Ix9hvi6@y{|MLI4*0oE^bp z*PnZ;1vYjtWL}=4&jRtNN|6_%zc>>Fnr4E&P&{)*m3^Jk$i-63nNt*R>t~S)t?e@H zD#*LVbjWMPGg%A%5xo)nRxI)o8~!7}W|FK6GNlL*sYAfrFp-}qOfHREum&0czXQDh z4C_H@{VnLUh^fMop}Op~G-c9hRt`lQrv@Z79`PuzSI z=WPY^U`Bk6Vz`QO*h|!=Sm<}hL5YaRADSdHVwFoB7S38X9-~)UvVL6^PE<35PbHGL z*%d%FyBx%#2vjGB8L>F7j%F#HzA|a!Yei;<%r+%6J185}bkPShS?Du&#>K9DHpL69 z1**v2?}I}KToZ+5mOU!?U0L)P0`r*?Xg@?4$#Wk01AaPYeP4mA-^?SYi3D;`S5HojXM+&n?zif%n2{|2HoGT_}>tr>&vsdC7+g#@P^ECdd!c zPb|MJ83TlWBQG9!>BlxhDu{OC4#5CH;N@4Dxor=7JyFb2lUbMLkb`;F4{qjx?$I`H zl@1o_9V5$|_+yo8EbaI_jccKOr6(K2T6|3rWrnyqkQZiU@=yT@Y-0cQ1SiU2zO2;s z1mirgqiCMNBWJVGia|3{P&&Y=B>mkvX{+4B@9^vWEe%@xi8geb;2|=;b$PxW!f;y` z`1>earw(@Wl{vl=pDk~u>msC70aXuJBD@s8F72!PNV+$x+5l8Q?xb{ z{P|k(ZMvy<0$_e7xEq6+^d&1Cx2Y%^h?7qz7B!Hnhv{YSEVk|!;Qn#XtP?Pynce<> zEEOC7y`y|Cc+S27up~Mq;GUEMS!2jg8T6*o(e~n6@s8`HIjXB!r1o(h+o28k36~r&iftUQGa$bI*jjz>bwIKSpiqVptLJ)I#WBI zD7pBQ`c-;7jh^!t8_1yVx<0E4XXqkNN^>xpK$MQ{qfZ~_Ko zNSrwhyQXZ7i}{yKvBA?^?9VI}gU%}I9wq95>|xiW(XeSZ_*zM}iPuU$kR9mI)a6Xx z48gb@-v_a<9Dlw4mP-{`p?`MCa2ExCMD&y ze*tzS#HTJZjF5OvYG6sFz>;io$EY2AOSKs^Who<5dcic$npICVCT){phVOAdOevYy zX=jx<)!S~}t9A@D)8Vv=C?xRKDn!9JX#lnh?ezo&kc-ez4|}tlhxwA2Ex<3%xw|Wx zdj$S=-y(}I#{Dc}Q0VsY?RME=p^@NQC0c+(1X%JDa#Uo3rK>W*gbYAf4#RGDG(P)ZjG_LhG27(5nT+0Nwcq&F*+G!<`dY z3037{Dbw32FwGbPOW;xdKUD_bnPAXl|0u_pfjz#kyl=7RpR_SHseb~*)BB_n#^?Bp z3wb}Pqa~(2V;6r?r2RPky5wUXm|sE~SDxA{ zY5w@bAIxi_&_{7l0N-ZwowxG8>IY$NWNB??i1%D*Tf1UrJdo}s>Z6IBdS(yQ zCd^jIrdz0C@t3=Dq~u}2Ej91IDV1*0K!IvHCX&c52Hv&XchJwvid+oR3qP2BvEau( zuU-Uebn6eQNf?;K!ak77k89J=r0#DnA-b(f#@-n+=>9_&MSFK>*Tv-a+o)IAu>$9AOgNt+I9-6fs&9bU9vGI-dkhxt`UjxuwyC#d1TN_<^g z;EohxJ$q5;x@&;DsvPFqFkMe*Sx3ytO8K-+8JjbP{})#>Q;FpA+wvu-db*Sb!g3U;#fd_0?K4pG8E8+YB zKaTJ06^6FO*Av0M*AwnlnWMO^5pNs*U_gh`{G|Er%_(CWgOdFbC_+=<#{brWv*X|t z@`pX-7($7&Q20>2f5WJfM1mqv>OHNvANJ&6myL0%gul*ucKeUog3g0{8@`nIspZ4R zNJF?og&`x)ST7zUo7Z0gj2LdTg=4s(w@_Qe2<{ z{FlBH5zOU}FrRyH|Jf*DISuFcRh+{L^Wj)@r_hAow{N|l%s7Pm5onvPgPfkCUkks6n#_m zAQEmz$${>c;5Z8N$r)znO|11nlZ8{m>bi_<>HuY;;>L|vC+E>(=`0o=W>w_f`{W-7 z;oaY;h#l5+l?N5R#(4qs-&TdK6+a_iL7X((a2*#_L=k9huNn-&g4)omDeUrmg`{W! zgZLj!R*&^y_Nu&aFgS)(=eyXP*rRwl#yVq=?a3@B_xP@2(Ea&MhuOenG(~$)~ zR}4%5lo_67WFgK#(UPcYxn=za;5`^-YJIz)82zy>M547DC_+t9DRE!&u^}I?`%r>p zVpqsGLb_M?m!=+F(UVdNv>X7|9eTD+QnCb}x`-&g%v2I53@@_aKkR*&lMNnNJ1|X- zjb&E*sK&kT=?9owuHN1& zK9&VyyQEl}6{ZYbD=sO@Lud5?%xqrP#^NyF{#ukT5Vk5)hBl#f79#*40kzi?CUP5K zWcIWtEFn~sT1eo=ozO!l1@s=C@3$Gh7D~PWG!V1HLJzp8A3HY?AK;~->+LOGe+)vZ z>I{l15)Fb5m}rf{3|czN;-^>Lf(RD_clS@vYF~>Ezx77k1ev zH@OcWZaTOi&dT=imp-utvJc%Ud<-i|fgQSRXUUg{S+y!Fy)!*(opw%ZLxpl zQ*5>DRhe&6y$lLOq6wE2Y!(j+8fE)UxC|?6fEWx(kC$EbKginXoD6Utc5@~5nfER^ zxqSXgUIGUi8w@P19_hohP-+|bWin}dacw?kmbv<@TCC~D0cAJ+Zq2eDUpx&2l$k!e1MEQa<}C#j86~e;Mr|}EtWS|9C7RbW%HO0p}!Bv zl;T(6H2bbD6YwC<=wy;<{7ev^g-pN4AK0kU&Cr?1ALSE|SUnVab7oj&j8q!|LrUAo ztU9>Ngg{1)eFd~r(j$5=Vv>>dFArTnJ7NE?w9{bNGUCL7v;(?zMC?)=2~B&Lv31Lm z5=j$4>Voi4te356`1^n+Z&JPZ*?pWi!Kd6zXPIN>p7U3y!LuS<=L>p*o~W;8W$HE+ zn=>G^j{i25h18uDUnf%b;C-ny$bFNfzIfPL9Oj|%hU4EDs4Z@aNyo|pU-5s3Z|-2U zAYDz~91=vb%`n0Dqwer5y`bzX_1qV?ro!7#jGGnKmz0-Bj3d_OmiL?V888`Z)4T^I zef^&MuhV=1f}hdl25D}?IIB@cs>cS;JDsM;=Y-(^J2Nih#`xx&$8@R}-`BFp`^W30 z#9VJ@&4=+&`!BaU6Y*nMvp2p6p-F_^PkwX`6Ahn^{^oYBO=*2<9E6;pNRF_2@92O!PbB49f@PWj0S|iWNrx`Z`nZOKBNj_Oq<{e^<9oy7uAB;c zA=X?#&M#C1ktsu!8KIA>S5WVBKZ%I8%$DuFG_Ceh=Z3By?7pQ$SMaL2|0xltJYGTr zThz#C7d9|d-xwXI3``w{Sr)?{;4qa{_3^vXq3(#DF>A1tg#h~)>I{@{f(7X@)1y4M ztcIny!1_QSk|Tz+EBOu$O0rA>luLh*25c(@*U+$srAxCMB-@a4W68Ib$@8YnaBIrDv7o``7={_kZSBa@h)m;?QtO%opsW^@+$_pCE$lXRRJ6a z+syaRBE$D!E@k0M7M9_msyaP|nV8kO(mtSgD`Qo*%<28GP57XpqE}sXqHVHf7ugFU zFM@BUlz10zRm}Wu{tX1hhGn4(KSOvwjyy}5KGi>F%FfLGR@Ic!@VEvLB{l=jibQ6c zX-~1K@e2g7@kU~SA%@n#k-W`2w8AQ|7%)A|75$06HaO$Rs1}mYQ%_TqHz9!8V1PkuPt5ivg6LY8ZPU@gGDuQ-{dNt>OIJQX#IWlusKdFIGUDL6Mh}3>dat zra%oCt^CH8Y}SU|n>{bRH7C4Xwp8OtAU;~Q2yvGR&wd2@L6sZ~(A zgE|OL2h=cP4^`0(SOR~tp!;lwoePVs7FS0nZb}``~t)K_5tC&QFNy3 zhSS18>!nH|ku~hz%T|hp%r}YQUqB##1}9aSiO7}KrbKEdyz4{5CpRh4gTACq8IA8p zrxcPd4H?6|P`_-oPDvhAs5kHTk0F|*&mV7 str: return f""" -This is the Kaggle scenario for the competition: {self.competition} +### Kaggle Agent: Automated Feature Engineering & Model Tuning Evolution + +#### [Overview](#_summary) + +In this scenario, our automated system proposes hypothesis, choose action, implements code, conducts validation, and utilizes feedback in a continuous, iterative process. + +#### Kaggle Competition info + +Current Competition: [{self.competition}](https://www.kaggle.com/competitions/{self.competition}) + +#### [Automated R&D](#_rdloops) + +- **[R (Research)](#_research)** +- Iteration of ideas and hypotheses. +- Continuous learning and knowledge construction. + +- **[D (Development)](#_development)** +- Evolving code generation, model refinement, and features generation. +- Automated implementation and testing of models/features. + +#### [Objective](#_summary) + +To automatically optimize performance metrics within the validation set or Kaggle Leaderboard, ultimately discovering the most efficient features and models through autonomous research and development. """ def get_scenario_all_desc(self, task: Task | None = None) -> str: diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index b9201d98..38ec0947 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -96,7 +96,7 @@ def crawl_descriptions(competition: str, wait: float = 3.0, force: bool = False) return descriptions -def download_data(competition: str, local_path: str = "/data/userdata/share/kaggle") -> None: +def download_data(competition: str, local_path: str = KAGGLE_IMPLEMENT_SETTING.local_data_path) -> None: data_path = f"{local_path}/{competition}" if not Path(data_path).exists(): subprocess.run(["kaggle", "competitions", "download", "-c", competition, "-p", data_path]) @@ -136,7 +136,7 @@ def score_rank(competition: str, score: float) -> tuple[int, float]: def download_notebooks( - competition: str, local_path: str = "/data/userdata/share/kaggle/notebooks", num: int = 15 + competition: str, local_path: str = f"{KAGGLE_IMPLEMENT_SETTING.local_data_path}/notebooks", num: int = 15 ) -> None: data_path = Path(f"{local_path}/{competition}") from kaggle.api.kaggle_api_extended import KaggleApi @@ -183,7 +183,9 @@ def notebook_to_knowledge(notebook_text: str) -> str: return response -def convert_notebooks_to_text(competition: str, local_path: str = "/data/userdata/share/kaggle/notebooks") -> None: +def convert_notebooks_to_text( + competition: str, local_path: str = f"{KAGGLE_IMPLEMENT_SETTING.local_data_path}/notebooks" +) -> None: data_path = Path(f"{local_path}/{competition}") converted_num = 0 @@ -219,7 +221,7 @@ def convert_notebooks_to_text(competition: str, local_path: str = "/data/userdat print(f"Converted {converted_num} notebooks to text files.") -def collect_knowledge_texts(local_path: str = "/data/userdata/share/kaggle") -> dict[str, list[str]]: +def collect_knowledge_texts(local_path: str = KAGGLE_IMPLEMENT_SETTING.local_data_path) -> dict[str, list[str]]: """ { "competition1": [ diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 64f16b52..d3104994 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -186,8 +186,6 @@ class KGDockerConf(DockerConf): # Path("git_ignore_folder/data").resolve(): "/root/.data/" # } - # local_data_path: str = "/data/userdata/share/kaggle" - # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3 class DockerEnv(Env[DockerConf]):