From 7d2e5ae941fbf95a4a65a6b146badeeaa790da19 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 8 May 2019 18:58:36 +0200 Subject: [PATCH] Allow lambda expression as cast rules using functions. Before that it was necessary to install a function in the lisp environment either in the source itself in src/utils/transforms.lisp, or in a lisp file loaded with --load-lisp-file (or -l for shorts). While this could be good enough, sometimes a very simple combination of existing features is required to transform a function and so doing some level of lisp coding directly in the load command is a nice to have. Fixes #961. --- src/parsers/command-cast-rules.lisp | 7 ++++++- src/utils/transforms.lisp | 10 +++++++++- test/data/DNORDOC.DBF | Bin 0 -> 14266 bytes test/data/DNORDOC.DBT | Bin 0 -> 53844 bytes test/dbf-memo.load | 12 ++++++++++++ 5 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 test/data/DNORDOC.DBF create mode 100644 test/data/DNORDOC.DBT create mode 100644 test/dbf-memo.load diff --git a/src/parsers/command-cast-rules.lisp b/src/parsers/command-cast-rules.lisp index 75fa7cb..eb200b3 100644 --- a/src/parsers/command-cast-rules.lisp +++ b/src/parsers/command-cast-rules.lisp @@ -141,7 +141,12 @@ (string (intern (string-upcase fname) :pgloader.transforms)) (symbol fname)))) -(defrule cast-function (and kw-using maybe-qualified-function-name) +(defrule transform-expression sexp + (:lambda (sexp) + (eval sexp))) + +(defrule cast-function (and kw-using (or maybe-qualified-function-name + transform-expression)) (:destructure (using symbol) (declare (ignore using)) symbol)) (defun fix-target-type (source target) diff --git a/src/utils/transforms.lisp b/src/utils/transforms.lisp index a87614f..65e9d0d 100644 --- a/src/utils/transforms.lisp +++ b/src/utils/transforms.lisp @@ -104,6 +104,7 @@ logical-to-boolean db3-trim-string db3-numeric-to-pgsql-numeric + db3-numeric-to-pgsql-integer db3-date-to-pgsql-date)) @@ -505,10 +506,17 @@ (defun db3-numeric-to-pgsql-numeric (value) "DB3 numerics should be good to go, but might contain spaces." - (let ((trimmed-string (string-right-trim '(#\Space) value))) + (let ((trimmed-string (string-trim '(#\Space) value))) (unless (string= "" trimmed-string) trimmed-string))) +(defun db3-numeric-to-pgsql-integer (value) + "DB3 numerics should be good to go, but might contain spaces." + (when value + (let ((integer-or-nil (parse-integer value :junk-allowed t))) + (when integer-or-nil + (write-to-string integer-or-nil))))) + (defun db3-date-to-pgsql-date (value) "Convert a DB3 date to a PostgreSQL date." (when (and value (string/= "" value) (= 8 (length value))) diff --git a/test/data/DNORDOC.DBF b/test/data/DNORDOC.DBF new file mode 100644 index 0000000000000000000000000000000000000000..5e080f4cfad1cf46c9f2592ea1fe77e75010d3ed GIT binary patch literal 14266 zcmcJWy^dsAR)tGKNC*iF41{$J+(V!LA9FN~5Ju-9G#HrY*TToGo*rQ!je2+l9)O49 zJ=);g5qWQ9Rb_|nWu`){tjvzgj2-9fz4lsb@BHE4|KzX#?Zbx;|NG%LeEjn9@~eOT z?%QAe@*ltb@Zp#8?B9O&Ej;(~i|_tk{_l74;m_s6n+MN-`-@-Sp66G5`0!_Mo|8}I z=l|vQllR~MS6@E=>wo>H+t0tqQ-ArVpMHbH_T4{7c<;ye7h6ste7JW>~>HT50lMnrOw;wt6-Mr+OdPOxxY0BHMR4Hkx(9#OaXYE|r1JL%=ujHGq zL|}Lc8-r1;hopS>TIEg{Da<~CTh+IDMBu(g1lcY(OvtL)5S6b{tLj`)eN8@3t(MNM zygvNbh+y`qbJRiE+?VnxJ5IwV)oaSJw%%HgdjL9ojR=elIvE$0^|o>gwyD}yP{#CW z%nUKdenb$zMg+_Dp7Mx_7o23`=&H;}I)*-)Gd;WcsD)&;F~;g(tSwKXG3H7w7!zD; zS}mKr@+~=)bBwBnJc9NM9TP=2+gs^LDIS#d=<2mgrR z+57%~N1#oMVFs(}EM(&JO=XiUrD6SAJ^0p+TQK^QmwEEVl{3&w*Px?Uu2`*nG)C1L zhgwxnpQ7`=jy)62SHZ--Y>dS?RbvK~3uRDjnJReQXL2S_b6}#erW8%-82_{?_?`=y zxJoL+_M&WWMtMV3fX9V&EGwH*txM)O1821^)B-@Qe)roKZ@uyF+|0!YtgfD9Q_)rW z!YEk*gHa{h%+zsFf9r_A^Oe?4`+EdASOpWML5>u&OS$)99@PjYl84$P&fxo(WPI`OHLGh8 zlvT&bmR4lw$$l0~1g?UK)417|9p!Nbq$b4>jQjZgrd{&6yo*uSB3PVrF1-EXXipwW zqS_!MVzbVV=6%bL2!IS7bk^mT1r+i7Fnuk85K@b2SdM*A3!wxM34ETxIdAG5bsd># zH4%92Yzg|~e{4T(T5}O!mgBVuhILfnq#HmK$OB)%gWA=I#pRl-o%Y}d7AVl3M2xQy z1ir_7Sq#@ASdK&B-Kx?O-PwUnIR*zvw_JPmYk=FY4|~T%=d^MD{R_DDifa)7{=PQv zROp3Th;FFn;jE*DtQ*}S#G^M_Meg`vLl^RZg7CV* z6J`g9+i?colngteB2}g)j+gxGlWP&!Nc(YAV_u60dLRNb;7HV32Y;u{T>kxG!9-(t z{UC_+!%HeY$3Ga6c%P1T1WOwhhN5%g4BAwtxtL;^mD)Ta028UjU~YZkH!s|VDSvQZ zu&Mdg=?gk7dZUvF9KtEM0agzQV-G-E!9;@8oWZRc-R;xQ?qJ)l-Gb?{Cu4^y-hfD` z1L_jdU@Ohh=0#~f&Hzjdj^0RPc<=Q0P_9MLXRc$$qQGD}Q@8Ln=Phl576~Kvb3P&f z6D1Ecec|4jsLFTp@md5r^i{eoO6M|QC=libV^OHNQkL~(hu^OO9FV>MRX6zw99+GH zkf{QD(~Y87LQ^g4sFt3=XX8lAqc1o^H3~l{RHF+&%hax&LD3V2PD{u$5kc*sBSehE zEVS)9TbKtTIdBM$6mKl;<40;C{Si3QY-9F9Unr(^6bs2vY7LSH)wB?+p-}&H!en+B8~~;Tp)By?v;67svycDn%0Fmo4##;o2GSGS;A$mG~ua zqz*p2ZXxe_wvKkL?O^LJN?*_clCE6L@1Ws5h-(p`g-u%+M@eL}cTtq9rY_fP$&9c} zKJdUjKm6kiuDLiQeltBKHC&)Ckp18R#I>2$*5XUTIr4xe1wSDZZF+_W+m{AgQg1@J2e|RX6Gcb|1Kx3d9o`i74#hC$AHW^PfHMrTdUX%kR z+3U#5RwU(-iT>G(eiF}>Jn*$5%(78)AR7Dx9cc(OYq~OmJ~lrdxA2}s`{b4PFRnyj zGcJwe?41|gm_ej6Muo7@{lY*m`26syxdkJt4M_U&hyCT=`S(f$v5m6yq6Xo{IGU(; zlmi?|hvneQWyN?sf-AKU5bqPmj#_{y@y5|iGZlrM2~_3gA!kAks^sfys`e&P?@s}b%s^HkE5QV9(0#k*Q&>=LwPQY1NhnwexD|-?|fBgM_ZA9Tr zd>0z8MKGI(7f&oqz=)iHsRD$GGHjgA%mMB8`Up?d>f1ZG)}7-R$P{j2qL;`-K2&Y8 z(#?)KaU0tKNTkcH79j2a{muV;`jVf0btMx=EvaH+C{m7FfTB|ieX2CV@Q%{R54Ikp zFW}+`tKp)4|Kn$`dwglaJr4{uZS#zhT96NB7OIfg3Z140bM@bzPf* zQXaa_t@kpF#yx_miKnV9h=iTRoC`R>Acz=e`x%I*`g#WbOayw49A+ZFDh$~BB^EyB zgmW|IiC2g?-$JvP>oRY&4HVFjlaeS1yP!h3#U36mPLMg%Jpz9!f>Hf+eUaRecIyL< zIfVxLt?6T{Kn-?ODt5t%0pw1eFj0XgR%Wg_q-1-fb?zr1677fpGI!z@Fvn`@0&>xD z2FxnRuN;`~!Ud+Qwjg|@IYY`ZO=6RoYaS{y0H#G-kqab#d?3;d6K}PE<8vZ{R#yV? zARjVA@e%JF8cd;KLwbjCT2n>ip`L@y*mr_Rw#m#3OJ%@_wRPZQJ=A*?uFNfH8);xC zA{g3H*_APF6q_t#5s~n?=utCb`)#9nw=j~xrsphgTT#AWV>VN z$v=Al8V<8Zz_pz?gKi;ShCT-d^UFsEcEl%&0gKwNm>Si%m?i;3+!7#;m)6KcOD!z; zD4-b&mAr=#1rzta@GOKKmEjPOeOky8_uf}1Il73P>=AJm8E+N4VJA)|&a~ss*wfSh zR8E=u!bhY(i*s~vBtqZM0H5c?8Nhr@t`U%{G$n=?aIXwHYW%7`0iamY7BZeCBS)Eh zi+Ue}SsV9Z#Zymg2CQ=Qg(#S~pTRR&-p?Q~Eu@|RfYL}ZuA3GN;sjbj&bc~?K&BR< z;A92734tZ_LKP72Bq+UOlBLHjJTQ^YapDZt;CEFKf|>r>X_?eDmwahsN^z{CScT6NPBL3@%2nNo0vy?%Sxw0>E*?}Vwm6qCY zrmnT6F6J?kk@BM;m^pC^&;>kY^3IN!A6{B~#Y6(bcsdaQU61)h7lTG{Ukq{Vzo{(# zAw#4(XEPgFSC=aU$D{$KN~|K2F!vxxhS?im?PmZO?wN>eI}rhtgjqWFi1jkOV55Lv z4jqjcvkn5xcj6#uiVkZL>V8CU=ZQCtbnXloRfz6T!?5uL zLt&^bYc>;`!erkYOEy~qdKyDLQbcxfJdRT9(jj8a1!>=4RYwF5OytH+oB{dAZWDIB zoAUOgB(djV0&aHTchV6-q=8*fIRgeQ!c-Zr4AwTyb}@B)>w0y zz%a`U(wTreeuu&uf;*YfQ3_ioC07LiG^I-(V5l`rEuv0sBA2*^gCV2`Ph>iJB7(sd zOZ0d`aS#UG8HSQb%vz~)N8=Gw7`L#8iYc?Y2fDR*>jO^W|3kHh7I~$J83Qwm8l`$)W>4B?f>`w%K!Wjyl{(v-)ET}ftF9=BU2ApH^ zAPM^#!3`6|$ugYAlS9~1nNenMsPxN@ts8~{MlH85mg z(nSOaDu&+(D*T204Bqg>XQJ)UOcqP36D z!|)jj#%VOt6&PUh5Fv{Xcb_@?j>?el6Spt`qR6CH4a?AkDLP=YSoGm)7mbj&Hx`;H zbW6IhjHjuEDw+wIfw{-hkr@;O^7ZwRADC!QMc^siP(+2|4|5A34sZx8Vb3hC9j2I3 zt`l5l9}d^*Hgkn5)4A1H%-IT+WFP8X@a&bTnp0ePCX|K(e;Fn-`; zGDcwM0LO46*O)jm;4u|F!G@760?9rS>zPREYSv>PDL;53=XoLzJcZyy7Bdzh50Ee9 z0k6e&&B&95HyKyKAG((@FXLy@Rb)$HC51Hzn9`(ap(2kncxY#L>K0fOfUXx1wu^GD zFdy0DqTW}C0|6iDjRJX5$EGyYiE3QDeVM7kU*Odp^~e-zzXid>SEl+@IPRFpJcx9k zl`$G1hdlygoCM@n!YYAhu$s7aC_sZF5*+ObP9(A!bfhn^9_7$&^??U(F!2;ap_r&S z7_woZPE;9ZYbiABv*HS#dh%l#q&N9*g$B}9odB4iGtWO z5o2M$1<>n61hjUz;ADukWUP*SV6fkXIy0^VZZl8grLpvbKP;>PJkT%{`~xo_&77Z( z4qH4T5KP=_;aLbrEp#@gSoxJ1v1n)UbEHu%Y-Ar7Ot{pIUX%-VHM9)h!gKc&3_`cA z)}}16YAWaTQQl(Wvqab4G4Ex2yQq^BbTaY!`6wA6XQ z=ws#@P%mx3T@Qv*9+(JjpST5h7EKX>5KsirAG$L@NRkEKWnuY`w!A@A+%h*$UnX4* zKL--eG5{!rxW?t!Gcj_#-aM+0!p* zUt|xgnw7mU3jghRz|@@x;NMVKAneEgJSJOTeBv%lJeeu=1(iX$Mj}`={z6j(55(_Y asO0?V^dFgFuzidD=EStk{kep)Kyb0&VzB^(6&)74e64iey<<(2L1jB263mW z5U<_{(nY3Yk*Rl7)Xsn}awEsOC`FvgEcVn4-*G6A z3N{-XGY*EmXkcT;=(NX?TL374}kvf54;sf3jQllqQ_M< z`D6U|80CXR2mc4DEts3YKNyq`Y6$p0NNvH~)Z~x&KNzr%4QcY{_|LBKNa9_-Ocbdc z*ULh=-5*YLR$(d7QLWw-1AlK&wIXPfKl}fC`+qEd=N}3F1sBi?hpsAZ^2hwYD-K;f zx5*#zUsoKudM@~PrTO)E?jrwthku0s>j7-VQZ)Hv{9mzp*S@+Y|Nox;Q$7Dj{J-`v zA0Qz3KR{^#-O}U_{~r)qr>20vlbV7pbo2Y)q{>sZo!diRXUhKk7r*limic_c{?-36 z0i0J)@YhZLSpN?m_(B7l=YMorR3&fuR#OV6quZ|z_31ChalWI;EV-tbIbZU4{@KRim>E%5-5QYy13wmi1Hlz6? z!#97qJuwXPS`T|Vj<@;T&ri?a{`#7BpU`!`-+waPk9PV1v>t$3mZH)B#n?#h^7*t> zNh}QKfXnoxqLWl7HXWSc17ZB}Or`ceo+DJG=v+?t^jQ5x^`3J9B>0g~%g9z*J@~Iw zo!(Vu`OCY6UWyJ5>6p88edNE+`LUg0DZo3d&H}%cAL`66$2yj!ps~y=4*YaXN7aW! zWsM;2p1dzRz5d=^-j~5}@9_X>1DyeMT(FV?(E^Js7P_>DZT(kpiVf~Wtn zG>Ljdg+5KiY>2py_FxhrFrRP3;R#io>cgDbR4XK)8buJi?pVwMA z*T;6hC*dT(Csfb=7gN6IlRsp_fng@cu{z`AkDTofwgCR&XkfEjGN0vjNI_| zw?32ye;+8dCnNZ`r&}Lt27ezYwI?I^x2IblY6gEFD77ae__wE9A8H1FA1JjaBlx$c zTOVq6=a2P2A9yY+Bls_?v9QX6e^@-1Z4LM@tFf@kyYt8XPgvww*Q!4y!QUTNZAj|QALD-;`tQ0< zcmDAIU9)QA`QYD%R{g0H{QY6ohNR%%hF1Nl6a4*Q)rO?t--cHGsneZ5=70WBb(a+U z-E|dWNq7F3|AT*s8rbYMq5p%w+x|cNe~6ZX{vRS$_cnpQyRJek>CPYX|IOx)@gMv{ z)WBx9$=&~>|A%NP?))+SZ#IAQfA9}c1DoBZBYvTu;Fo=;^;@84F90rjd5>Obqc7?= zf#Gdl0h`FT#1X*!``_6X?7QC<-urx4VoI;|iM1GhN1UWWrdRbvuKCj1(05x~e*-2I BEzAG_ literal 0 HcmV?d00001 diff --git a/test/dbf-memo.load b/test/dbf-memo.load new file mode 100644 index 0000000..497b7ca --- /dev/null +++ b/test/dbf-memo.load @@ -0,0 +1,12 @@ +LOAD DBF + FROM data/DNORDOC.DBF with encoding cp866 + INTO postgresql:///pgloader + TARGET TABLE public.dnordoc + WITH truncate, create table, disable triggers + + cast + column dnordoc.normdocid + to uuid + using (lambda (normdocid) + (empty-string-to-null (right-trim normdocid))), + column dnordoc.doctype to integer using db3-numeric-to-pgsql-integer;