use md5 in cache table; test ~1800 ACSII chars

NikolayS · Oct 24, 2016 · 96e768a · 96e768a
1 parent 80c304b
commit 96e768a
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 6 deletions.
diff --git a/install_core.sql b/install_core.sql
@@ -13,16 +13,18 @@ create schema google_translate;
 -- to avoid reaching 2K limit for URL in Google API calls.
 -- "Regular" urlencode() with multibyte chars support is shown above (commented out block of code). 
 create or replace function google_translate.urlencode(text) returns text as $$
-    select 
+    select
         string_agg(
             case
                 when ascii(ch) in (32, 160) then -- spaces, CR, LF
                     '+'
+                when ascii(ch) between 127 and 165 then -- unsupported chars
+                    '+'
                 when ol=1 and (ch ~ '[+\]\[%&#]+' or ascii(ch) < 32)  -- this is not traditional urlencode!
                     then regexp_replace(upper(substring(ch::bytea::text, 3)), '(..)', E'%\\1', 'g')
-                else 
+                else
                     ch
-            end, 
+            end,
             ''
         )
     from (
@@ -32,13 +34,15 @@ create or replace function google_translate.urlencode(text) returns text as $$
 $$ language sql immutable strict;
 
 create table google_translate.cache(
+    id bigserial primary key,
     source char(2) not null,
     target char(2) not null,
     q text not null,
     result text not null,
-    created timestamp not null default now(),
-    primary key(q, source, target)
+    created timestamp not null default now()
 );
+create unique index u_cache_q_source_target on google_translate.cache
+    using btree(md5(q), source, target);
 
 comment on table google_translate.cache is 'Cache for Google Translate API calls';
 
@@ -77,7 +81,7 @@ begin
             trim(queries.q) as q
         from
             google_translate.cache
-        right join queries on trim(queries.q) = cache.q
+        right join queries on md5(trim(queries.q)) = md5(cache.q)
             and cache.source = translate.source
             and cache.target = translate.target
     loop

diff --git a/test/cases/api_chars_128_165.expected b/test/cases/api_chars_128_165.expected
@@ -0,0 +1,5 @@
+ google_translate 
+------------------
+ test:
+(1 row)
+
diff --git a/test/cases/api_chars_128_165.sh b/test/cases/api_chars_128_165.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+psql -U postgres test -c "select v1.google_translate('ru', 'en', 'test: ' || string_agg(char, '')) from (select chr(generate_series) as char from generate_series(127, 165)) as z"
diff --git a/test/cases/api_chars_1_127.expected b/test/cases/api_chars_1_127.expected
@@ -0,0 +1,5 @@
+                                                          google_translate                                                          
+------------------------------------------------------------------------------------------------------------------------------------
+ !. &quot;# $% &amp; &#39;() * +, - / 0123456789:; &lt;=&gt; @ ABCDEFGHIJKLMNOPQRSTUVWXYZ [\\] ^ _` abcdefghijklmnopqrstuvwxyz | ~?
+(1 row)
+
diff --git a/test/cases/api_chars_1_127.sh b/test/cases/api_chars_1_127.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+psql -U postgres test -c "select v1.google_translate('ru', 'en', string_agg(char, '')) from (select chr(generate_series) as char from generate_series(1, 126)) as z"
diff --git a/test/cases/api_chars_1_1810.expected b/test/cases/api_chars_1_1810.expected
@@ -0,0 +1,5 @@
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   google_translate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ !. &quot;# $% &amp; &#39;() * +, - / 0123456789:; &lt;=&gt; @ ABCDEFGHIJKLMNOPQRSTUVWXYZ [\\] ^ _` abcdefghijklmnopqrstuvwxyz | ~ |§¨ © ª «¬®¯ ° ± ²³&#39;μ¶ · ¸? ¹º »¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ × ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö ÷ øùúûüýþÿĀāĂăĄąĆćĈĉĊċČčĎďĐđĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħĨĩĪīĬĭĮįİıĲĳĴĵĶķĸĹĺĻļĽľĿŀŁłŃńŅņŇňŉŊŋŌōŎŏŐőŒœŔŕŖŗŘřŚśŜŝŞşŠšŢţŤťŦŧŨũŪūŬŭŮůŰűŲųŴŵŶŷŸŹźŻżŽžsƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏƐƑƒƓƔƕƖƗƘƙƚƛƜƝƞƟƠơƢƣƤƥƦƧƨΣƪƫƬƭƮƯưƱƲƳƴƵƶƷƸƹƺƻƼƽƾƿ|ǁǂ!ǄǅǆǇǈǉǊǋǌǍǎǏǐǑǒǓǔǕǖǗǘǙǚǛǜǝǞǟǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯǰǱǲǳǴǵǶǷǸǹǺǻǼǽǾǿȀȁȂȃȄȅȆȇȈȉȊȋȌȍȎȏȐȑȒȓȔȕȖȗŞşŢţȜȝȞȟȠȡȢȣȤȥȦȧȨȩȪȫȬȭȮȯȰȱȲȳȴȵȶȷȸȹȺȻȼȽȾȿɀɁɂɃɄɅɆɇɈɉɊɋɌɍɎɏɐɑɒɓɔɕɖɗɘəɚɛɜɝɞɟɠɡɢɣɤɥɦɧɨɩɪɫɬɭɮɯɰɱɲɳɴɵɶɷɸɹɺɻɼɽɾɿʀʁʂʃʄʅʆʇʈʉʊʋʌʍʎʏʐʑʒʓʔʕʖʗʘʙʚʛʜʝʞʟʠʡʢʣʤʥʦʧʨʩʪʫʬʭʮʯʰʱʲʳʴʵʶʷʸ&#39;&quot;&#39;&#39;&#39;&#39;&#39;ˀˁ˂˃˅ˉ&#39;&#39;ˌˎˏːˑ˒˓˔˕˖˗˘˙˚˛~˝˞˟ˠˡˢˣˤ˥˦˧˨˩˪˫ˬ˭ˮ˯˰˱˲˳˴˵˶˷˸ ˹˺˻˼˽˾˿̴̵̶̷̸̡̢̧̨̛̖̗̘̙̜̝̞̟̠̣̤̥̦̩̪̫̬̭̮̯̰̳̹̺̻̼͇͈͉͍͎̄̅̆̇̈̉̊̋̌̍̏̐̑̒̓̔̽̾̿̓̈͆͊͋͌̕̚͏͓͔͕͖͙͚͐͑͒͗͛ͣͤͥͦͧͨͩͪͫͬͭͮͯ͘͜͟͢͝͞͠͡ͰͱͲͳ&#39;͵Ͷͷ͸͹ͻͼͽ; Ϳ΀΁΂΃Ά · ΈΉΊ΋Ό΍ΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡ΢ΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώϏβθΥΎΫφπ&amp;ϘϙϚςϜϝϞϟϠϡϢϣϤϥϦϧϨϩϪϫϬϭϮϯκρςϳΘε϶ϷϸΣϺϻϼϽϾϿЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏABVGDEZhZIYKLMNOPRSTUFHTsChShSchYEYuYaabvgdezhziyklmnoprstufhtschshschyeyuyaѐёђѓєѕіїјљњћќѝўџѠѡѢѣѤѥѦѧѨѩѪѫѬѭѮѯѰѱѲѳѴѵѶѷѸѹѺѻѼѽѾѿҀҁ҂҃҄҅҆҇҈҉ҊҋҌҍҎҏҐґҒғҔҕҖҗҘҙҚқҜҝҞҟҠҡҢңҤҥҦҧҨҩҪҫҬҭҮүҰұҲҳҴҵҶҷҸҹҺһҼҽҾҿӀӁӂӃӄӅӆӇӈӉӊӋӌӍӎӏӐӑӒӓӔӕӖӗӘәӚӛӜӝӞӟӠӡӢӣӤӥӦӧӨөӪӫӬӭӮӯӰӱӲӳӴӵӶӷӸӹӺӻӼӽӾӿԀԁԂԃԄԅԆԇԈԉԊԋԌԍԎԏԐԑԒԓԔԕԖԗԘԙԚԛԜԝԞԟԠԡԢԣԤԥԦԧԨԩԪԫԬԭԮԯ ԰ԱԲԳԴԵԶԷԸԹԺԻԼԽԾԿՀՁՂՃՄՅՆՇՈՉՊՋՌՍՎՏՐՑՒՓՔՕՖ՗՘՜, ՠաբգդեզէըթժիլխծկհձղճմյնշոչպջռսվտրցւփքօֆեւֈ.- ֋֌֍֎֏ ֐ְֱֲֳִֵֶַָֹֺֻּֽ֑֖֛֢֣֤֥֦֧֪֚֭֮֒֓֔֕֗֘֙֜֝֞֟֠֡֨֩֫֬֯-ֿ|ׁׂ:ׅׄ׆ׇ׈׉׊׋׌׍׎׏אבגדהוזחטיךכלםמןנסעףפץצקרשת׫׬׭׮ׯוווייי&#39;&quot;׵׶׷׸׹׺׻׼׽׾׿ \u0600\u0601\u0602\u0603؄؅؆ ؇؈؉؊؋, /؎؏ؘؙؚؖؗ;? ؜؝* ؠءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿفقكلمنهوىيٟ% .,*ٮٯٱٲٳٴاٴوٴۇٴيٴٹٺٻټٽپٿڀځڂڃڄڅچڇڈډڊڋڌڍڎڏڐڑڒړڔڕږڗژڙښڛڜڝڞڟڠڡڢڣڤڥڦڧڨکڪګڬڭڮگڰڱڲڳڴڵڶڷڸڹںڻڼڽھڿۀہۂۃۄۅۆۇۈۉۊۋیۍێۏېۑےۓ.ە\u06dd۞۩ۮۯ 0123456789 0123456789 ۺۻۼ۽۾ۿ ܀܁܂܃܄܅܆܇܈܉܊܋܌܍܎\u070fܐܑܒ
+(1 row)
+
diff --git a/test/cases/api_chars_1_1810.sh b/test/cases/api_chars_1_1810.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+psql -U postgres test -c "select v1.google_translate('ru', 'en', string_agg(char, '')) from (select chr(generate_series) as char from generate_series(1, 1810)) as z"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/bin/bash

		psql -U postgres test -c "select v1.google_translate('ru', 'en', 'test: ' \|\| string_agg(char, '')) from (select chr(generate_series) as char from generate_series(127, 165)) as z"