From 7fbfd0087bb964c0e9e2ed71cb0a165b333a319d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 10 Sep 2014 20:58:19 +0200 Subject: [PATCH 001/582] updates #12 --- lokingyql/lokingyql.py | 9 ++++++++- lokingyql/yahooauth.py | 7 +++++++ tests/tests.py | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lokingyql/lokingyql.py b/lokingyql/lokingyql.py index 05bc2a4..c0b5f9c 100644 --- a/lokingyql/lokingyql.py +++ b/lokingyql/lokingyql.py @@ -17,6 +17,7 @@ class LokingYQL(object): - community : set to to have access to community tables ''' default_url = 'https://query.yahooapis.com/v1/public/yql' + oauth_url = 'http://query.yahooapis.com/v1/yql' community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table def __init__(self, table=None, url=default_url, community=False, format='json', oauth=None): @@ -24,6 +25,7 @@ def __init__(self, table=None, url=default_url, community=False, format='json', self.table = table self.format = format self._query = None # used to build query when using methods such as + + + + + ![CDATA[ // Include the flickr signing library + y.include("http:blog.pipes.yahoo.net/wp-content/uploads/flickr.js"); + // GET the flickr result using a signed url + var fs = new flickrSigner(api_key,secret); + response.object = y.rest(fs.createUrl({method:method, format:""})).get().response(); + ]] + + + + + + + + ![CDATA[ console.log('hello this is an insert function'); ]] + + + + diff --git a/lokingyql/contrib/opentable/tests_data/mytable.xml b/lokingyql/contrib/opentable/tests_data/mytable.xml new file mode 100644 index 0000000..ca17498 --- /dev/null +++ b/lokingyql/contrib/opentable/tests_data/mytable.xml @@ -0,0 +1,25 @@ + + + + josuebrunel + http://josuebrunel.org/api + http://josuebrunel.org/doc.html + + SELECT * FROM mytable + + + + +
diff --git a/lokingyql/contrib/opentable/tests_data/titi.xml b/lokingyql/contrib/opentable/tests_data/titi.xml new file mode 100644 index 0000000..61f4809 --- /dev/null +++ b/lokingyql/contrib/opentable/tests_data/titi.xml @@ -0,0 +1,11 @@ + + + + josuebrunel + http://josuebrunel.org/api + http://josuebrunel.org/doc.html + + SELECT * FROM mytable + + +
diff --git a/lokingyql/contrib/opentable/tests_data/toto.xml b/lokingyql/contrib/opentable/tests_data/toto.xml new file mode 100644 index 0000000..61f4809 --- /dev/null +++ b/lokingyql/contrib/opentable/tests_data/toto.xml @@ -0,0 +1,11 @@ + + + + josuebrunel + http://josuebrunel.org/api + http://josuebrunel.org/doc.html + + SELECT * FROM mytable + + +
From ca8027a25499063dc79b1f8c720af13901ad312a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 11 Mar 2015 10:33:07 -0500 Subject: [PATCH 081/582] cleaning the cat house --- .../contrib/opentable/tests_data/after.xml | 20 ----------- .../contrib/opentable/tests_data/before.xml | 33 ------------------- .../contrib/opentable/tests_data/mytable.xml | 25 -------------- .../contrib/opentable/tests_data/titi.xml | 11 ------- .../contrib/opentable/tests_data/toto.xml | 11 ------- 5 files changed, 100 deletions(-) delete mode 100644 lokingyql/contrib/opentable/tests_data/after.xml delete mode 100644 lokingyql/contrib/opentable/tests_data/before.xml delete mode 100644 lokingyql/contrib/opentable/tests_data/mytable.xml delete mode 100644 lokingyql/contrib/opentable/tests_data/titi.xml delete mode 100644 lokingyql/contrib/opentable/tests_data/toto.xml diff --git a/lokingyql/contrib/opentable/tests_data/after.xml b/lokingyql/contrib/opentable/tests_data/after.xml deleted file mode 100644 index e41dfde..0000000 --- a/lokingyql/contrib/opentable/tests_data/after.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - josuebrunel - http://josuebrunel.org/api - http://josuebrunel.org/doc.html - - SELECT * FROM mytable - - - - - - - - ![CDATA[ console.log('hello this is an insert function'); ]] - - - -
diff --git a/lokingyql/contrib/opentable/tests_data/before.xml b/lokingyql/contrib/opentable/tests_data/before.xml deleted file mode 100644 index 535828b..0000000 --- a/lokingyql/contrib/opentable/tests_data/before.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - josuebrunel - http://josuebrunel.org/api - http://josuebrunel.org/doc.html - - SELECT * FROM mytable - - - - - - - - - ![CDATA[ console.log('hello this is an insert function'); ]] - - - -
diff --git a/lokingyql/contrib/opentable/tests_data/mytable.xml b/lokingyql/contrib/opentable/tests_data/mytable.xml deleted file mode 100644 index ca17498..0000000 --- a/lokingyql/contrib/opentable/tests_data/mytable.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - josuebrunel - http://josuebrunel.org/api - http://josuebrunel.org/doc.html - - SELECT * FROM mytable - - - - -
diff --git a/lokingyql/contrib/opentable/tests_data/titi.xml b/lokingyql/contrib/opentable/tests_data/titi.xml deleted file mode 100644 index 61f4809..0000000 --- a/lokingyql/contrib/opentable/tests_data/titi.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - josuebrunel - http://josuebrunel.org/api - http://josuebrunel.org/doc.html - - SELECT * FROM mytable - - -
diff --git a/lokingyql/contrib/opentable/tests_data/toto.xml b/lokingyql/contrib/opentable/tests_data/toto.xml deleted file mode 100644 index 61f4809..0000000 --- a/lokingyql/contrib/opentable/tests_data/toto.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - josuebrunel - http://josuebrunel.org/api - http://josuebrunel.org/doc.html - - SELECT * FROM mytable - - -
From cce2cf8c49c1b18bf3038233bd6cd8d213d5fad3 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 11 Mar 2015 10:51:48 -0500 Subject: [PATCH 082/582] fix self.bindings --- lokingyql/contrib/opentable/yqltable.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lokingyql/contrib/opentable/yqltable.py b/lokingyql/contrib/opentable/yqltable.py index 1f2e25f..d4eeaa2 100755 --- a/lokingyql/contrib/opentable/yqltable.py +++ b/lokingyql/contrib/opentable/yqltable.py @@ -19,6 +19,7 @@ def __init__(self, name, author, apiKeyURL, documentationURL, sampleQuery, descr self.sampleQuery = sampleQuery self.table_attr = table_attr self.etree = self._init_table_elementTree() + self.bindings = bindings if bindings: [ self.addBinder(binder) for binder in bindings ] From eddc7cc6c0a42e3d7ceda727f022e741aec58833 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 12 Mar 2015 07:35:55 -0500 Subject: [PATCH 083/582] test_remove_function added #26 --- lokingyql/contrib/opentable/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index f4de7b1..8781a3e 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -79,6 +79,12 @@ def test_add_function_from_file(self,): self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) print self.xml_pretty_print(self.binder.etree) + def test_remove_function(self,): + self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) + print self.xml_pretty_print(self.binder.etree) + self.assertEqual(self.binder.removeFunction(), True) + print self.xml_pretty_print(self.binder.etree) + def test_save_file(self,): self.table.save() self.assertEquals(os.path.isfile('mytest.xml'),True) From d5a50b0e82f526c4c68b515f11932ad0dc409140 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 12 Mar 2015 07:36:29 -0500 Subject: [PATCH 084/582] fix #26 : remove function from binder fully functional --- lokingyql/contrib/opentable/binder.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index 7285bf3..8d792df 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -87,6 +87,21 @@ def addFunction(self, function_code, from_file=''): t_execute.text = "\n ![CDATA[ {0} ]] \n".format(function_code) return True + + def removeFunction(self,): + """Removes function of the binder + """ + root = self.etree + t_execute = root.find('execute') + + try: + root.remove(t_execute) + return True + except Exception, e: + print(e) + + return False + class BinderKey(object): """Class representing a key which is part of inputs From 1ae852a1646d0a77b6264da9b4c4d9f716172abc Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 12 Mar 2015 08:36:07 -0500 Subject: [PATCH 085/582] updates #29 test_add_function_table --- lokingyql/contrib/opentable/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index 8781a3e..073bc78 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -126,6 +126,11 @@ def test_create_table_with_binder(self,): table.save(name='mytable', path='tests_data') self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + def test_add_function_table(self): + self.xml_pretty_print(self.table.etree) + self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) + self.xml_pretty_print(self.table.etree) + def tearUp(self): os.path.unlink('tests_data/mytest.xml') os.path.unlink('tests_data/toto.xml') From 96827a6d3b5af16ab3ca41c2d45c4ad5f0c6725f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 12 Mar 2015 08:39:15 -0500 Subject: [PATCH 086/582] fix #29 : add function to table OK --- lokingyql/contrib/opentable/tests.py | 4 ++-- lokingyql/contrib/opentable/yqltable.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index 073bc78..8dbfd4d 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -127,9 +127,9 @@ def test_create_table_with_binder(self,): self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) def test_add_function_table(self): - self.xml_pretty_print(self.table.etree) + print(self.xml_pretty_print(self.table.etree)) self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) - self.xml_pretty_print(self.table.etree) + print(self.xml_pretty_print(self.table.etree)) def tearUp(self): os.path.unlink('tests_data/mytest.xml') diff --git a/lokingyql/contrib/opentable/yqltable.py b/lokingyql/contrib/opentable/yqltable.py index d4eeaa2..ef02223 100755 --- a/lokingyql/contrib/opentable/yqltable.py +++ b/lokingyql/contrib/opentable/yqltable.py @@ -127,3 +127,19 @@ def removeBinder(self, name): return True return False + + def addFunction(self, func_code, from_file=None): + """Adds function to a YQL Table + """ + + if from_file: + with open(from_file) as f: + func_code = f.read() + + root = self.etree + + t_execute = xtree.SubElement(root, 'execute') + t_execute.text = "\n [!CDATA[ {0} ]]\n".format(func_code) + + return True + From 6e6234ec88229dd2a1a9ecba3bf8fad737ebf84c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 12 Mar 2015 08:42:22 -0500 Subject: [PATCH 087/582] #30 : test_remove_function_table added --- lokingyql/contrib/opentable/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index 8dbfd4d..57f1cd3 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -131,6 +131,13 @@ def test_add_function_table(self): self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) print(self.xml_pretty_print(self.table.etree)) + def test_remove_function_table(self,): + print(self.xml_pretty_print(self.table.etree)) + self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) + print(self.xml_pretty_print(self.table.etree)) + self.assertEquals(self.table.removeFunction(),True) + print(self.xml_pretty_print(self.table.etree)) + def tearUp(self): os.path.unlink('tests_data/mytest.xml') os.path.unlink('tests_data/toto.xml') From 64efdd278bdda5f1cb6ab2c8722c6df429700705 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 12 Mar 2015 08:53:03 -0500 Subject: [PATCH 088/582] fix #30 : remove function from table OK --- lokingyql/contrib/opentable/yqltable.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lokingyql/contrib/opentable/yqltable.py b/lokingyql/contrib/opentable/yqltable.py index ef02223..f406163 100755 --- a/lokingyql/contrib/opentable/yqltable.py +++ b/lokingyql/contrib/opentable/yqltable.py @@ -143,3 +143,18 @@ def addFunction(self, func_code, from_file=None): return True + def removeFunction(self): + """Removes from a table + """ + + root = self.etree + + t_execute = root.find('execute') + + try : + root.remove(t_execute) + return True + except Execption,e: + print(e) + + return False From 9f50c20ee18d069517337012359eadd85cd7a2d7 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 12 Mar 2015 09:12:53 -0500 Subject: [PATCH 089/582] #28 test_add_paging OK --- lokingyql/contrib/opentable/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index 57f1cd3..df9e487 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -85,6 +85,12 @@ def test_remove_function(self,): self.assertEqual(self.binder.removeFunction(), True) print self.xml_pretty_print(self.binder.etree) + def test_add_paging(self,): + paging = BinderPage('page', start={'id': 'ItemPage', 'default': '1'}, pageSize={'id':'Count' ,'max':'25'}, total={'default': '10'}) + print self.xml_pretty_print(self.binder.etree) + self.assertEquals(self.binder.addPaging(paging), True) + print self.xml_pretty_print(self.binder.etree) + def test_save_file(self,): self.table.save() self.assertEquals(os.path.isfile('mytest.xml'),True) From 41b8c17584e82a5e1f387ebd471112b9d0aa187d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 12 Mar 2015 19:39:49 +0100 Subject: [PATCH 090/582] #28 : Paging class added --- lokingyql/contrib/opentable/binder.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index 8d792df..574d908 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -131,3 +131,25 @@ def to_elementTree(self,): """ return self.etree +class BinderPage(object): + + def __init__(self, model, start={}, pageSize={}, total={}): + """Class representing a binder Page + """ + self.model = model + self.start = start + self.pageSize = pageSize + + self.etree = self.__buildElementTree() + + def __buildElementTree(self,): + """Turns object into into an ElementTree + """ + t_paging = xtree.Element('paging') + + for item in self.__dict__.items(): + t_paging.set(*item) + + return t_paging + + From 9bbeb05176b411fa36f1c8d982cffcf22a7f3117 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 07:54:30 +0100 Subject: [PATCH 091/582] BinderPage added --- lokingyql/contrib/opentable/binder.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index 574d908..7728bc8 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -133,7 +133,7 @@ def to_elementTree(self,): class BinderPage(object): - def __init__(self, model, start={}, pageSize={}, total={}): + def __init__(self, model, start, pageSize, total): """Class representing a binder Page """ self.model = model @@ -147,9 +147,16 @@ def __buildElementTree(self,): """ t_paging = xtree.Element('paging') - for item in self.__dict__.items(): - t_paging.set(*item) - + # for key in self.__dict__.keys(): + # xtree.SubElement(t_paging, key).set(*self.__dict__[key]) + + # I would go for something like + for key in self.__dict__.keys(): + if key != 'model': + t_tag = xtree.SubElement(t_paging, key) + for item in self.__dict__[key].items() : + t_tag.set(*item) + return t_paging From f599b9dab385899e601dd13b41e593425d283974 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 07:55:14 +0100 Subject: [PATCH 092/582] BinderPage test OK --- lokingyql/contrib/opentable/tests.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index df9e487..9644236 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -3,7 +3,7 @@ import unittest from xml.dom import minidom from xml.etree import cElementTree as xtree -from binder import Binder, BinderKey +from binder import Binder, BinderKey, BinderPage from yqltable import YqlTable import readline, rlcompleter @@ -86,10 +86,14 @@ def test_remove_function(self,): print self.xml_pretty_print(self.binder.etree) def test_add_paging(self,): - paging = BinderPage('page', start={'id': 'ItemPage', 'default': '1'}, pageSize={'id':'Count' ,'max':'25'}, total={'default': '10'}) - print self.xml_pretty_print(self.binder.etree) - self.assertEquals(self.binder.addPaging(paging), True) - print self.xml_pretty_print(self.binder.etree) + start= {'id': 'ItemPage', 'default': '1'} + pageSize= {'id':'Count' ,'max':'25'} + total= {'default': '10'} + #paging = BinderPage('page', start={'id': 'ItemPage', 'default': '1'}, pageSize={'id':'Count' ,'max':'25'}, total={'default': '10'}) + paging = BinderPage('page', start, pageSize, total) + print self.xml_pretty_print(paging.etree) + #self.assertEquals(self.binder.addPaging(paging), True) + #print self.xml_pretty_print(self.binder.etree) def test_save_file(self,): self.table.save() From 3dcc4167f10fe1d9cc6488d2c018d22644f460aa Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 09:07:48 +0100 Subject: [PATCH 093/582] #28: total taken into account --- lokingyql/contrib/opentable/binder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index 7728bc8..9e226d4 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -139,6 +139,7 @@ def __init__(self, model, start, pageSize, total): self.model = model self.start = start self.pageSize = pageSize + self.total = total self.etree = self.__buildElementTree() From 403fe77f35ed6d79f1d5d8ee8169a47a46f33371 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 09:20:37 +0100 Subject: [PATCH 094/582] updates #28: tests updated --- lokingyql/contrib/opentable/tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index 9644236..7c73d23 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -89,11 +89,10 @@ def test_add_paging(self,): start= {'id': 'ItemPage', 'default': '1'} pageSize= {'id':'Count' ,'max':'25'} total= {'default': '10'} - #paging = BinderPage('page', start={'id': 'ItemPage', 'default': '1'}, pageSize={'id':'Count' ,'max':'25'}, total={'default': '10'}) paging = BinderPage('page', start, pageSize, total) - print self.xml_pretty_print(paging.etree) - #self.assertEquals(self.binder.addPaging(paging), True) - #print self.xml_pretty_print(self.binder.etree) + print self.xml_pretty_print(self.binder.etree) + self.assertEquals(self.binder.addPaging(paging), True) + print self.xml_pretty_print(self.binder.etree) def test_save_file(self,): self.table.save() From 9b2bbb2bbf0afc47dde6bd283fd69e1ffc092044 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 09:21:18 +0100 Subject: [PATCH 095/582] updates #28 : fix adding issue --- lokingyql/contrib/opentable/binder.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index 9e226d4..e52fccf 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -102,6 +102,24 @@ def removeFunction(self,): return False + def addPaging(self, paging): + """Adds paging to binder + """ + + root = self.etree + t_paging = root.find('paging') + + if not t_paging: + t_paging = xtree.SubElement(root, 'paging') + + try: + root.append(paging.etree) + return True + except Exception, e: + print(e) + + return False + class BinderKey(object): """Class representing a key which is part of inputs @@ -147,11 +165,7 @@ def __buildElementTree(self,): """Turns object into into an ElementTree """ t_paging = xtree.Element('paging') - - # for key in self.__dict__.keys(): - # xtree.SubElement(t_paging, key).set(*self.__dict__[key]) - # I would go for something like for key in self.__dict__.keys(): if key != 'model': t_tag = xtree.SubElement(t_paging, key) From 389e32ffa89dbe692425df8da994af1abedf81b6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 03:42:01 -0500 Subject: [PATCH 096/582] fix #28: Add paging to binder OK --- lokingyql/contrib/opentable/binder.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index e52fccf..b28f533 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -105,12 +105,7 @@ def removeFunction(self,): def addPaging(self, paging): """Adds paging to binder """ - root = self.etree - t_paging = root.find('paging') - - if not t_paging: - t_paging = xtree.SubElement(root, 'paging') try: root.append(paging.etree) From ac5fa954c59328560b129c33582a954d1c6b213f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 03:42:32 -0500 Subject: [PATCH 097/582] import of every class in the package root --- lokingyql/contrib/opentable/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lokingyql/contrib/opentable/__init__.py b/lokingyql/contrib/opentable/__init__.py index 8073d26..a887d03 100755 --- a/lokingyql/contrib/opentable/__init__.py +++ b/lokingyql/contrib/opentable/__init__.py @@ -1 +1,2 @@ from yqltable import * +from binder import * From 0d12954b2689155160ce005405b8e2daf35effdc Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 03:48:15 -0500 Subject: [PATCH 098/582] #31: removePaging from binder test added --- lokingyql/contrib/opentable/tests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index 7c73d23..b92fa6f 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -94,6 +94,17 @@ def test_add_paging(self,): self.assertEquals(self.binder.addPaging(paging), True) print self.xml_pretty_print(self.binder.etree) + def test_remove_paging(self,): + start= {'id': 'ItemPage', 'default': '1'} + pageSize= {'id':'Count' ,'max':'25'} + total= {'default': '10'} + paging = BinderPage('page', start, pageSize, total) + print self.xml_pretty_print(self.binder.etree) + self.assertEquals(self.binder.addPaging(paging), True) + print self.xml_pretty_print(self.binder.etree) + self.assertEqual(self.binder.removePaging(), True) + print self.xml_pretty_print(self.binder.etree) + def test_save_file(self,): self.table.save() self.assertEquals(os.path.isfile('mytest.xml'),True) From 2a89b4b50fdb68ecda20fe4b7dc54d9288a19d6f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 03:51:29 -0500 Subject: [PATCH 099/582] #31: Remove paging from Binder done, test left --- lokingyql/contrib/opentable/binder.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index b28f533..e282920 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -114,7 +114,21 @@ def addPaging(self, paging): print(e) return False - + + def removePaging(self,): + """Removes paging from Binder + """ + root = self.etree + t_paging = root.find('paging') + + try: + root.remove(t_paging) + return True + except Exception, e: + print(e) + + return False + ` class BinderKey(object): """Class representing a key which is part of inputs From 64e69921df4320b7e98cc3b33a2fc146df92399f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 03:52:54 -0500 Subject: [PATCH 100/582] fix #31: remove paging from Binder OK --- lokingyql/contrib/opentable/binder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index e282920..26ae052 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -128,7 +128,7 @@ def removePaging(self,): print(e) return False - ` + class BinderKey(object): """Class representing a key which is part of inputs From 2ae9ee1562de53c33dc30cfdec142d4a0b7ae688 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 18:27:29 +0100 Subject: [PATCH 101/582] Binder can now be instantiated with paging --- lokingyql/contrib/opentable/binder.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index 26ae052..5fc300d 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -9,7 +9,7 @@ class Binder(object): inputs : list of BinderKey object """ - def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[], inputs=[]): + def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[], inputs=[], paging=None): """Initializes the class """ self.name = name @@ -20,10 +20,17 @@ def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[] # Builds the element tree self.etree = self._buildElementTree() - # Adding inpust passed as parameters + # Adding inputs passed as parameters if inputs: [ self.addInput(key.etree) for key in inputs ] + # Adding paging + if paging: + [ self.addPaging(page) for page in pages ] + + if paging: + self.addPaging(pagin) + def __repr__(self): return "".format(self.name) From 31a2a2b0e2194752d2312200f47baf9050e44d47 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 18:28:01 +0100 Subject: [PATCH 102/582] updating test suite --- lokingyql/contrib/opentable/tests.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index b92fa6f..16661e0 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -40,6 +40,11 @@ def setUp(self,): self.key = BinderKey(**self.key_desc) self.key2 = BinderKey(id='song', type='xs:string', paramType='path', required='true') + start= {'id': 'ItemPage', 'default': '1'} + pageSize= {'id':'Count' ,'max':'25'} + total= {'default': '10'} + self.paging = BinderPage('page', start, pageSize, total) + def xml_pretty_print(self, data): """Pretty print xml data """ @@ -86,21 +91,13 @@ def test_remove_function(self,): print self.xml_pretty_print(self.binder.etree) def test_add_paging(self,): - start= {'id': 'ItemPage', 'default': '1'} - pageSize= {'id':'Count' ,'max':'25'} - total= {'default': '10'} - paging = BinderPage('page', start, pageSize, total) print self.xml_pretty_print(self.binder.etree) - self.assertEquals(self.binder.addPaging(paging), True) + self.assertEquals(self.binder.addPaging(self.paging), True) print self.xml_pretty_print(self.binder.etree) def test_remove_paging(self,): - start= {'id': 'ItemPage', 'default': '1'} - pageSize= {'id':'Count' ,'max':'25'} - total= {'default': '10'} - paging = BinderPage('page', start, pageSize, total) print self.xml_pretty_print(self.binder.etree) - self.assertEquals(self.binder.addPaging(paging), True) + self.assertEquals(self.binder.addPaging(self.paging), True) print self.xml_pretty_print(self.binder.etree) self.assertEqual(self.binder.removePaging(), True) print self.xml_pretty_print(self.binder.etree) From 38ed97e77f9e50ff7c0520814795f3995876e7ac Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 18:30:38 +0100 Subject: [PATCH 103/582] Add url to binder test added --- lokingyql/contrib/opentable/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index 16661e0..d4e9cf9 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -102,6 +102,11 @@ def test_remove_paging(self,): self.assertEqual(self.binder.removePaging(), True) print self.xml_pretty_print(self.binder.etree) + def test_add_url(self,): + url = 'http://josuebrunel.org/service.js' + self.assertEquals(self.binder.addUrl(url), True) + print self.xml_pretty_print(self.binder.xtree) + def test_save_file(self,): self.table.save() self.assertEquals(os.path.isfile('mytest.xml'),True) From 7ed2393a8424e8337ec8b0a6ae2cf41df44e86a4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 18:36:51 +0100 Subject: [PATCH 104/582] fix add url to binder --- lokingyql/contrib/opentable/binder.py | 14 ++++++++++++++ lokingyql/contrib/opentable/tests.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index 5fc300d..a156bfc 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -45,6 +45,20 @@ def _buildElementTree(self,): return t_binder + def addUrl(self, url): + """Adds url to binder + """ + root = self.etree + + t_urls = root.find('urls') + if not t_urls: + t_urls = xtree.SubElement(root, 'urls') + + t_url = xtree.SubElement(t_urls, 'url') + t_url.text = url + + return True + def addInput(self, key): """Add key element to the binder """ diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index d4e9cf9..451bdc1 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -105,7 +105,7 @@ def test_remove_paging(self,): def test_add_url(self,): url = 'http://josuebrunel.org/service.js' self.assertEquals(self.binder.addUrl(url), True) - print self.xml_pretty_print(self.binder.xtree) + print self.xml_pretty_print(self.binder.etree) def test_save_file(self,): self.table.save() From c086e12036b2d4a184f6fc2ee7f5002101bbc059 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 18:39:34 +0100 Subject: [PATCH 105/582] remove url test added --- lokingyql/contrib/opentable/tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/opentable/tests.py index 451bdc1..88f02e0 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/opentable/tests.py @@ -107,6 +107,15 @@ def test_add_url(self,): self.assertEquals(self.binder.addUrl(url), True) print self.xml_pretty_print(self.binder.etree) + def test_remove_url(self,): + url = 'http://josuebrunel.org/service.js' + url2 = 'http://google.com' + self.assertEquals(self.binder.addUrl(url), True) + self.assertEquals(self.binder.addUrl(url2), True) + print self.xml_pretty_print(self.binder.etree) + self.assertEquals(self.binder.removeUrl(url), True) + print self.xml_pretty_print(self.binder.etree) + def test_save_file(self,): self.table.save() self.assertEquals(os.path.isfile('mytest.xml'),True) From e1d3c303a03742ce5b4c6637f4b5f2fbbdb95f20 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 13 Mar 2015 19:03:56 +0100 Subject: [PATCH 106/582] remove url from binder OK --- lokingyql/contrib/opentable/binder.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/opentable/binder.py index a156bfc..449839c 100755 --- a/lokingyql/contrib/opentable/binder.py +++ b/lokingyql/contrib/opentable/binder.py @@ -59,6 +59,20 @@ def addUrl(self, url): return True + def removeUrl(self, url): + """Removes a specified url of a binder + """ + root = self.etree + t_urls = root.find('urls') + if not t_urls: + return False + for t_url in t_urls.findall('url'): + if t_url.text == url.strip(): + t_urls.remove(t_url) + return True + + return False + def addInput(self, key): """Add key element to the binder """ From 652c2f44c6e85369fd543cbbde25355d86ceec82 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 17 Mar 2015 08:30:12 +0100 Subject: [PATCH 107/582] Default yqltable xmlns added --- lokingyql/contrib/opentable/yqltable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lokingyql/contrib/opentable/yqltable.py b/lokingyql/contrib/opentable/yqltable.py index f406163..78490ef 100755 --- a/lokingyql/contrib/opentable/yqltable.py +++ b/lokingyql/contrib/opentable/yqltable.py @@ -6,7 +6,7 @@ class YqlTable(object): """Class representating a YQL Table """ - _TAB_ATTR = {'xmlns':'', 'securityLevel':'any', 'https':'false'} + _TAB_ATTR = {'xmlns':'http://query.yahooapis.com/v1/schema/table.xsd', 'securityLevel':'any', 'https':'false'} def __init__(self, name, author, apiKeyURL, documentationURL, sampleQuery, description=None, table_attr=None, bindings=[]): """Initialize the class From 0ea6fc89fa5de63e8adacd11515f35e25b7be442 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 17 Mar 2015 09:08:18 +0100 Subject: [PATCH 108/582] fix #34 : additional options added for payloader --- lokingyql/lokingyql.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lokingyql/lokingyql.py b/lokingyql/lokingyql.py index 63d8400..73e7950 100755 --- a/lokingyql/lokingyql.py +++ b/lokingyql/lokingyql.py @@ -20,7 +20,7 @@ class LokingYQL(object): oauth_url = 'http://query.yahooapis.com/v1/yql' community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table - def __init__(self, table=None, url=default_url, community=False, format='json', oauth=None): + def __init__(self, table=None, url=default_url, community=False, format='json', jsonCompact=False, crossProduct=False, debug=False, oauth=None): self.url = url self.table = table self.format = format @@ -29,6 +29,10 @@ def __init__(self, table=None, url=default_url, community=False, format='json', self.diagnostics = False # Who knows, someone would like to turn it ON lol self.limit = '' self.community = community # True means access to community data + self.format = format + self.crossProduct = crossProduct + self.jsonCompact = jsonCompact + self.debug = debug self.oauth = oauth self.yoauth = yahooauth.YahooOAuth() @@ -46,7 +50,10 @@ def payloadBuilder(self, query, format='json'): 'q' : query, 'callback' : '', #This is not javascript 'diagnostics' : self.diagnostics, - 'format' : format + 'format' : format, + 'crossProduct': self.crossProduct, + 'debug': self.debug, + 'jsonCompact': self.jsonCompact } self._payload = payload From 802da418edc0f5336cd8d09410ecee8199736e5b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 17 Mar 2015 09:11:43 +0100 Subject: [PATCH 109/582] disabling oauth --- lokingyql/lokingyql.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lokingyql/lokingyql.py b/lokingyql/lokingyql.py index 73e7950..85214be 100755 --- a/lokingyql/lokingyql.py +++ b/lokingyql/lokingyql.py @@ -33,8 +33,6 @@ def __init__(self, table=None, url=default_url, community=False, format='json', self.crossProduct = crossProduct self.jsonCompact = jsonCompact self.debug = debug - self.oauth = oauth - self.yoauth = yahooauth.YahooOAuth() def __repr__(self): '''Returns information on the current instance @@ -116,23 +114,23 @@ def buildResponse(self, response): return response - def loadConfig(self, module): - '''Loads OAuth config (consumer_key, consumer_secret) from module - ''' - try: - config_module = importlib.import_module(module) - except Exception, e: - raise errors.NoConfigFileError(e) + # def loadConfig(self, module): + # '''Loads OAuth config (consumer_key, consumer_secret) from module + # ''' + # try: + # config_module = importlib.import_module(module) + # except Exception, e: + # raise errors.NoConfigFileError(e) - try: - ck = config_module.consumer_key - cs = config_module.consumer_secret - except Exception, e: - raise errors.NoConfigParameter(e) + # try: + # ck = config_module.consumer_key + # cs = config_module.consumer_secret + # except Exception, e: + # raise errors.NoConfigParameter(e) - self.yoauth = yahooauth.YahooOAuth(ck, cs) + # self.yoauth = yahooauth.YahooOAuth(ck, cs) - return config_module + # return config_module ###################################################### # @@ -254,3 +252,4 @@ def showTables(self, format='json'): response = self.executeQuery(payload) return response + From 8770c16c88bb6bbce7363a5ad5fb27e00639cad6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 17 Mar 2015 10:53:57 -0500 Subject: [PATCH 110/582] fix issue related to crossProduct --- lokingyql/contrib/opentable/__init__.py | 4 ++-- lokingyql/lokingyql.py | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lokingyql/contrib/opentable/__init__.py b/lokingyql/contrib/opentable/__init__.py index a887d03..99e276c 100755 --- a/lokingyql/contrib/opentable/__init__.py +++ b/lokingyql/contrib/opentable/__init__.py @@ -1,2 +1,2 @@ -from yqltable import * -from binder import * +import yqltable +import binder diff --git a/lokingyql/lokingyql.py b/lokingyql/lokingyql.py index 85214be..85a38b9 100755 --- a/lokingyql/lokingyql.py +++ b/lokingyql/lokingyql.py @@ -20,7 +20,7 @@ class LokingYQL(object): oauth_url = 'http://query.yahooapis.com/v1/yql' community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table - def __init__(self, table=None, url=default_url, community=False, format='json', jsonCompact=False, crossProduct=False, debug=False, oauth=None): + def __init__(self, table=None, url=default_url, community=False, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): self.url = url self.table = table self.format = format @@ -49,10 +49,11 @@ def payloadBuilder(self, query, format='json'): 'callback' : '', #This is not javascript 'diagnostics' : self.diagnostics, 'format' : format, - 'crossProduct': self.crossProduct, - 'debug': self.debug, - 'jsonCompact': self.jsonCompact + 'debug': self.debug, + 'jsonCompact': self.jsonCompact } + if self.crossProduct: + payload['crossProduct'] = self.crossProduct self._payload = payload @@ -78,8 +79,6 @@ def rawQuery(self, query, format='', pretty=False): def executeQuery(self, payload): '''Execute the query and returns and response''' - if self.oauth : - self.url = self.oauth_url response = requests.get(self.url, params= payload) From a5fc13150dae2bb071abfbc0f6322a10a0dc8f2c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 17 Mar 2015 11:04:39 -0500 Subject: [PATCH 111/582] cleaning the cat house --- lokingyql/db/models.py | 0 lokingyql/utils/logger.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100755 lokingyql/db/models.py delete mode 100755 lokingyql/utils/logger.py diff --git a/lokingyql/db/models.py b/lokingyql/db/models.py deleted file mode 100755 index e69de29..0000000 diff --git a/lokingyql/utils/logger.py b/lokingyql/utils/logger.py deleted file mode 100755 index e69de29..0000000 From 622f4d8755c141bae1ba06e24df59103b800ecd5 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 17 Mar 2015 11:23:42 -0500 Subject: [PATCH 112/582] refactoring and renaming files --- lokingyql/__init__.py | 3 ++- lokingyql/contrib/__init__.py | 4 ++-- lokingyql/contrib/opentable/__init__.py | 2 -- lokingyql/contrib/table/__init__.py | 2 ++ lokingyql/contrib/{opentable => table}/binder.py | 0 .../contrib/{opentable => table}/run_test.sh | 2 +- .../{opentable/yqltable.py => table/table.py} | 4 ++-- lokingyql/contrib/{opentable => table}/tests.py | 8 ++++---- .../{opentable => table}/tests_data/jscode.js | 0 tests/tests.py | 16 ++++++++-------- 10 files changed, 21 insertions(+), 20 deletions(-) delete mode 100755 lokingyql/contrib/opentable/__init__.py create mode 100755 lokingyql/contrib/table/__init__.py rename lokingyql/contrib/{opentable => table}/binder.py (100%) rename lokingyql/contrib/{opentable => table}/run_test.sh (57%) rename lokingyql/contrib/{opentable/yqltable.py => table/table.py} (98%) rename lokingyql/contrib/{opentable => table}/tests.py (97%) rename lokingyql/contrib/{opentable => table}/tests_data/jscode.js (100%) diff --git a/lokingyql/__init__.py b/lokingyql/__init__.py index 7e9a6e7..077cb64 100755 --- a/lokingyql/__init__.py +++ b/lokingyql/__init__.py @@ -1,2 +1,3 @@ -from lokingyql import LokingYQL +import contrib import errors +from lokingyql import LokingYQL diff --git a/lokingyql/contrib/__init__.py b/lokingyql/contrib/__init__.py index 18a604b..b27bbfe 100755 --- a/lokingyql/contrib/__init__.py +++ b/lokingyql/contrib/__init__.py @@ -1,2 +1,2 @@ -from yahooauth import * -from opentable import * +import yahooauth +import table diff --git a/lokingyql/contrib/opentable/__init__.py b/lokingyql/contrib/opentable/__init__.py deleted file mode 100755 index 99e276c..0000000 --- a/lokingyql/contrib/opentable/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import yqltable -import binder diff --git a/lokingyql/contrib/table/__init__.py b/lokingyql/contrib/table/__init__.py new file mode 100755 index 0000000..c6d1c4f --- /dev/null +++ b/lokingyql/contrib/table/__init__.py @@ -0,0 +1,2 @@ +import table +import binder diff --git a/lokingyql/contrib/opentable/binder.py b/lokingyql/contrib/table/binder.py similarity index 100% rename from lokingyql/contrib/opentable/binder.py rename to lokingyql/contrib/table/binder.py diff --git a/lokingyql/contrib/opentable/run_test.sh b/lokingyql/contrib/table/run_test.sh similarity index 57% rename from lokingyql/contrib/opentable/run_test.sh rename to lokingyql/contrib/table/run_test.sh index 1e87d60..33883f0 100755 --- a/lokingyql/contrib/opentable/run_test.sh +++ b/lokingyql/contrib/table/run_test.sh @@ -4,4 +4,4 @@ else method='' fi -python -m unittest tests.TestYqlTable$method +python -m unittest tests.TestTable$method diff --git a/lokingyql/contrib/opentable/yqltable.py b/lokingyql/contrib/table/table.py similarity index 98% rename from lokingyql/contrib/opentable/yqltable.py rename to lokingyql/contrib/table/table.py index 78490ef..14247d6 100755 --- a/lokingyql/contrib/opentable/yqltable.py +++ b/lokingyql/contrib/table/table.py @@ -2,7 +2,7 @@ from xml.dom import minidom from xml.etree import cElementTree as xtree -class YqlTable(object): +class Table(object): """Class representating a YQL Table """ @@ -25,7 +25,7 @@ def __init__(self, name, author, apiKeyURL, documentationURL, sampleQuery, descr [ self.addBinder(binder) for binder in bindings ] def __repr__(self,): - return "".format(self.name) + return "".format(self.name) def _xml_pretty_print(self, data): """Pretty print xml data diff --git a/lokingyql/contrib/opentable/tests.py b/lokingyql/contrib/table/tests.py similarity index 97% rename from lokingyql/contrib/opentable/tests.py rename to lokingyql/contrib/table/tests.py index 88f02e0..3057cb0 100755 --- a/lokingyql/contrib/opentable/tests.py +++ b/lokingyql/contrib/table/tests.py @@ -4,12 +4,12 @@ from xml.dom import minidom from xml.etree import cElementTree as xtree from binder import Binder, BinderKey, BinderPage -from yqltable import YqlTable +from table import Table import readline, rlcompleter readline.parse_and_bind('tab: complete') -class TestYqlTable(unittest.TestCase): +class TestTable(unittest.TestCase): def setUp(self,): self.table_desc = { @@ -20,7 +20,7 @@ def setUp(self,): 'sampleQuery': 'SELECT * FROM mytable', } - self.table = YqlTable(**self.table_desc) + self.table = Table(**self.table_desc) self.binder_desc = { 'name': 'select', @@ -153,7 +153,7 @@ def test_create_table_with_binder(self,): self.binder.addInput(self.key) self.binder.addFunction('', from_file='tests_data/jscode.js') self.table_desc['bindings'] = [self.binder] - table = YqlTable(**self.table_desc) + table = Table(**self.table_desc) table.save(name='mytable', path='tests_data') self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) diff --git a/lokingyql/contrib/opentable/tests_data/jscode.js b/lokingyql/contrib/table/tests_data/jscode.js similarity index 100% rename from lokingyql/contrib/opentable/tests_data/jscode.js rename to lokingyql/contrib/table/tests_data/jscode.js diff --git a/tests/tests.py b/tests/tests.py index bd935ae..1497e2b 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -2,7 +2,7 @@ from lokingyql import LokingYQL from lokingyql.errors import NoTableSelectedError -from lokingyql.contrib import YahooOAuth +#from lokingyql.contrib import YahooOAuth from test_config import consumer_key, consumer_secret @@ -10,7 +10,7 @@ class LokingYqlTest(unittest.TestCase): def setUp(self,): self.yql = LokingYQL() - self.yoauth = YahooOAuth(consumer_key, consumer_secret) + #self.yoauth = YahooOAuth(consumer_key, consumer_secret) def tearUp(self,): pass @@ -23,13 +23,13 @@ def test_use(self, url= 'http://myserver.com/mytables.xml'): # def test_select_where(self, table='geo.states', items=['']) - def test_config(self,): - '''Tests test config file - ''' - conf = self.yql.loadConfig('tests.test_config') +#def test_config(self,): + # '''Tests test config file + # ''' + # conf = self.yql.loadConfig('tests.test_config') - self.assertEquals('josue', conf.consumer_key) - self.assertEquals('brunel', conf.consumer_secret) + # self.assertEquals('josue', conf.consumer_key) + # self.assertEquals('brunel', conf.consumer_secret) #def test_desc_with_no_table(self,): #self.assertRaises(NoTableSelectedError, self.yql.desc()) From 56d0103564bbed6ad55dbb25805f0259f810a31a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 1 Apr 2015 22:41:46 +0200 Subject: [PATCH 113/582] now using find_packages --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0a57d8b..18d6747 100755 --- a/setup.py +++ b/setup.py @@ -12,8 +12,8 @@ author = "Josue Kouka", author_email = "josuebrunel@gmail.com", url = "https://github.com/josuebrunel/lokingYQL", - packages = ['lokingyql'], + packages = find_packages, platforms=['Any'], license='BSD', install_requires = required -) \ No newline at end of file +) From 760a32e0d60e7cbfa5dcead782716a17a1cc9960 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 2 Apr 2015 11:26:56 +0200 Subject: [PATCH 114/582] fix typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 18d6747..350b4f3 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ author = "Josue Kouka", author_email = "josuebrunel@gmail.com", url = "https://github.com/josuebrunel/lokingYQL", - packages = find_packages, + packages = find_packages(), platforms=['Any'], license='BSD', install_requires = required From b6c9e972449c4ef73f5d27d04e285871afbdfb2d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 2 Apr 2015 13:19:42 +0200 Subject: [PATCH 115/582] changing import from table --- lokingyql/contrib/table/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lokingyql/contrib/table/__init__.py b/lokingyql/contrib/table/__init__.py index c6d1c4f..ca3c7c4 100755 --- a/lokingyql/contrib/table/__init__.py +++ b/lokingyql/contrib/table/__init__.py @@ -1,2 +1,2 @@ -import table -import binder +from table import * +from binder import * From 3b379794438f61693c554c414a6b1172d0cbe127 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 4 Apr 2015 22:04:18 +0200 Subject: [PATCH 116/582] #36 : tests added --- lokingyql/contrib/table/binder.py | 443 +++++++++++++++--------------- lokingyql/contrib/table/tests.py | 2 + 2 files changed, 224 insertions(+), 221 deletions(-) diff --git a/lokingyql/contrib/table/binder.py b/lokingyql/contrib/table/binder.py index 449839c..cc24e72 100755 --- a/lokingyql/contrib/table/binder.py +++ b/lokingyql/contrib/table/binder.py @@ -1,221 +1,222 @@ -from xml.etree import cElementTree as xtree - -class Binder(object): - """Class describing binders : select, insert, update, delete - name : select, insert, update, delete - itemPath : dotted path i.e : products.product - produces : json or xml - urls : list of urls related to the api - inputs : list of BinderKey object - """ - - def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[], inputs=[], paging=None): - """Initializes the class - """ - self.name = name - self.itemPath = itemPath - self.pollingFrequencySeconds = str(pollingFrequencySeconds) - self.produces = produces - - # Builds the element tree - self.etree = self._buildElementTree() - - # Adding inputs passed as parameters - if inputs: - [ self.addInput(key.etree) for key in inputs ] - - # Adding paging - if paging: - [ self.addPaging(page) for page in pages ] - - if paging: - self.addPaging(pagin) - - def __repr__(self): - return "".format(self.name) - - def _buildElementTree(self,): - """Builds ElementTree out of Binder object - """ - t_binder = xtree.Element(self.name) - - for item in self.__dict__.items(): - if item[0] != 'name': - t_binder.set(*item) - - return t_binder - - def addUrl(self, url): - """Adds url to binder - """ - root = self.etree - - t_urls = root.find('urls') - if not t_urls: - t_urls = xtree.SubElement(root, 'urls') - - t_url = xtree.SubElement(t_urls, 'url') - t_url.text = url - - return True - - def removeUrl(self, url): - """Removes a specified url of a binder - """ - root = self.etree - t_urls = root.find('urls') - if not t_urls: - return False - for t_url in t_urls.findall('url'): - if t_url.text == url.strip(): - t_urls.remove(t_url) - return True - - return False - - def addInput(self, key): - """Add key element to the binder - """ - root = self.etree - - t_input = root.find('inputs') - - if not t_input : - t_input = xtree.SubElement(root, 'inputs') - - t_input.append(key.etree) - - return True - - def removeInput(self, key_id): - """Removes an input from a binder - """ - root = self.etree - t_inputs = root.find('inputs') - - keys = t_inputs.findall('key') - - key = [ key for key in keys if key.get('id') == key_id ] - - try: - t_inputs.remove(key[0]) - return True - except Exception, e: - print(e) - return False - - def addFunction(self, function_code, from_file=''): - """Adds function section to the binder - """ - root = self.etree - - t_execute = root.find('execute') - - if not t_execute: - t_execute = xtree.SubElement(root, 'execute') - - if from_file : - with open(from_file) as f: - function_code = f.read() - - #t_execute.text = function_code - t_execute.text = "\n ![CDATA[ {0} ]] \n".format(function_code) - - return True - - def removeFunction(self,): - """Removes function of the binder - """ - root = self.etree - t_execute = root.find('execute') - - try: - root.remove(t_execute) - return True - except Exception, e: - print(e) - - return False - - def addPaging(self, paging): - """Adds paging to binder - """ - root = self.etree - - try: - root.append(paging.etree) - return True - except Exception, e: - print(e) - - return False - - def removePaging(self,): - """Removes paging from Binder - """ - root = self.etree - t_paging = root.find('paging') - - try: - root.remove(t_paging) - return True - except Exception, e: - print(e) - - return False - - -class BinderKey(object): - """Class representing a key which is part of inputs - """ - - def __init__(self, id, type, paramType, required='false', like=''): - """Initializes the class - """ - self.id = id - self.type = type - self.paramType = paramType - self.required = required - - self.etree = self._buildElementTree() - - def _buildElementTree(self,): - """Turns object into ElementTre - """ - t_key = xtree.Element('key') - for item in self.__dict__.items(): - t_key.set(*item) - - return t_key - - def to_elementTree(self,): - """Returns object as ElementTree - """ - return self.etree - -class BinderPage(object): - - def __init__(self, model, start, pageSize, total): - """Class representing a binder Page - """ - self.model = model - self.start = start - self.pageSize = pageSize - self.total = total - - self.etree = self.__buildElementTree() - - def __buildElementTree(self,): - """Turns object into into an ElementTree - """ - t_paging = xtree.Element('paging') - - for key in self.__dict__.keys(): - if key != 'model': - t_tag = xtree.SubElement(t_paging, key) - for item in self.__dict__[key].items() : - t_tag.set(*item) - - return t_paging - - +from xml.etree import cElementTree as xtree + +class Binder(object): + """Class describing binders : select, insert, update, delete + name : select, insert, update, delete + itemPath : dotted path i.e : products.product + produces : json or xml + urls : list of urls related to the api + inputs : list of BinderKey object + """ + + def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[], inputs=[], paging=None): + """Initializes the class + """ + self.name = name + self.itemPath = itemPath + self.pollingFrequencySeconds = str(pollingFrequencySeconds) + self.produces = produces + + # Builds the element tree + self.etree = self._buildElementTree() + + # Adding urls + self.urls = urls + + # Adding inputs passed as parameters + if inputs: + self.inputs = [ self.addInput(key.etree) for key in inputs ] + + # Adding paging + if paging: + self.paging = paging + self.addPaging(paging) + + def __repr__(self): + return "".format(self.name) + + def _buildElementTree(self,): + """Builds ElementTree out of Binder object + """ + t_binder = xtree.Element(self.name) + + for item in self.__dict__.items(): + if item[0] != 'name': + t_binder.set(*item) + + return t_binder + + def addUrl(self, url): + """Adds url to binder + """ + root = self.etree + + t_urls = root.find('urls') + if not t_urls: + t_urls = xtree.SubElement(root, 'urls') + + t_url = xtree.SubElement(t_urls, 'url') + t_url.text = url + + return True + + def removeUrl(self, url): + """Removes a specified url of a binder + """ + root = self.etree + t_urls = root.find('urls') + if not t_urls: + return False + for t_url in t_urls.findall('url'): + if t_url.text == url.strip(): + t_urls.remove(t_url) + return True + + return False + + def addInput(self, key): + """Add key element to the binder + """ + root = self.etree + + t_input = root.find('inputs') + + if not t_input : + t_input = xtree.SubElement(root, 'inputs') + + t_input.append(key.etree) + + return True + + def removeInput(self, key_id): + """Removes an input from a binder + """ + root = self.etree + t_inputs = root.find('inputs') + + keys = t_inputs.findall('key') + + key = [ key for key in keys if key.get('id') == key_id ] + + try: + t_inputs.remove(key[0]) + return True + except Exception, e: + print(e) + return False + + def addFunction(self, function_code, from_file=''): + """Adds function section to the binder + """ + root = self.etree + + t_execute = root.find('execute') + + if not t_execute: + t_execute = xtree.SubElement(root, 'execute') + + if from_file : + with open(from_file) as f: + function_code = f.read() + + #t_execute.text = function_code + t_execute.text = "\n ![CDATA[ {0} ]] \n".format(function_code) + + return True + + def removeFunction(self,): + """Removes function of the binder + """ + root = self.etree + t_execute = root.find('execute') + + try: + root.remove(t_execute) + return True + except Exception, e: + print(e) + + return False + + def addPaging(self, paging): + """Adds paging to binder + """ + root = self.etree + + try: + root.append(paging.etree) + return True + except Exception, e: + print(e) + + return False + + def removePaging(self,): + """Removes paging from Binder + """ + root = self.etree + t_paging = root.find('paging') + + try: + root.remove(t_paging) + return True + except Exception, e: + print(e) + + return False + + +class BinderKey(object): + """Class representing a key which is part of inputs + """ + + def __init__(self, id, type, paramType, required='false', like=''): + """Initializes the class + """ + self.id = id + self.type = type + self.paramType = paramType + self.required = required + + self.etree = self._buildElementTree() + + def _buildElementTree(self,): + """Turns object into ElementTre + """ + t_key = xtree.Element('key') + for item in self.__dict__.items(): + t_key.set(*item) + + return t_key + + def to_elementTree(self,): + """Returns object as ElementTree + """ + return self.etree + +class BinderPage(object): + + def __init__(self, model, start, pageSize, total): + """Class representing a binder Page + """ + self.model = model + self.start = start + self.pageSize = pageSize + self.total = total + + self.etree = self.__buildElementTree() + + def __buildElementTree(self,): + """Turns object into into an ElementTree + """ + t_paging = xtree.Element('paging') + + for key in self.__dict__.keys(): + if key != 'model': + t_tag = xtree.SubElement(t_paging, key) + for item in self.__dict__[key].items() : + t_tag.set(*item) + + return t_paging + + diff --git a/lokingyql/contrib/table/tests.py b/lokingyql/contrib/table/tests.py index 3057cb0..c62038b 100755 --- a/lokingyql/contrib/table/tests.py +++ b/lokingyql/contrib/table/tests.py @@ -93,6 +93,7 @@ def test_remove_function(self,): def test_add_paging(self,): print self.xml_pretty_print(self.binder.etree) self.assertEquals(self.binder.addPaging(self.paging), True) + print(self.binder.paging) print self.xml_pretty_print(self.binder.etree) def test_remove_paging(self,): @@ -105,6 +106,7 @@ def test_remove_paging(self,): def test_add_url(self,): url = 'http://josuebrunel.org/service.js' self.assertEquals(self.binder.addUrl(url), True) + print(self.binder.urls) print self.xml_pretty_print(self.binder.etree) def test_remove_url(self,): From ba9692f5ea7720e16290472e70dd9913571c008c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 4 Apr 2015 22:04:50 +0200 Subject: [PATCH 117/582] fix #36 : get url, paging and inputs from binder object --- lokingyql/contrib/table/binder.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lokingyql/contrib/table/binder.py b/lokingyql/contrib/table/binder.py index cc24e72..b11df58 100755 --- a/lokingyql/contrib/table/binder.py +++ b/lokingyql/contrib/table/binder.py @@ -16,12 +16,12 @@ def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[] self.itemPath = itemPath self.pollingFrequencySeconds = str(pollingFrequencySeconds) self.produces = produces - + self.urls = urls + self.inputs = inputs + self.paging = paging # Builds the element tree self.etree = self._buildElementTree() - # Adding urls - self.urls = urls # Adding inputs passed as parameters if inputs: @@ -29,7 +29,6 @@ def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[] # Adding paging if paging: - self.paging = paging self.addPaging(paging) def __repr__(self): @@ -41,7 +40,7 @@ def _buildElementTree(self,): t_binder = xtree.Element(self.name) for item in self.__dict__.items(): - if item[0] != 'name': + if item[0] not in ('name', 'inputs', 'urls', 'paging'): t_binder.set(*item) return t_binder @@ -49,6 +48,7 @@ def _buildElementTree(self,): def addUrl(self, url): """Adds url to binder """ + self.urls.append(url) root = self.etree t_urls = root.find('urls') @@ -77,6 +77,7 @@ def removeUrl(self, url): def addInput(self, key): """Add key element to the binder """ + self.inputs.append(key) root = self.etree t_input = root.find('inputs') @@ -141,6 +142,7 @@ def removeFunction(self,): def addPaging(self, paging): """Adds paging to binder """ + self.paging = paging root = self.etree try: From 074c1d37c400aa0e2d69498c6f7c588cab94eee0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 4 Apr 2015 22:09:07 +0200 Subject: [PATCH 118/582] fix #38: it's done --- lokingyql/contrib/table/binder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lokingyql/contrib/table/binder.py b/lokingyql/contrib/table/binder.py index b11df58..89d35cb 100755 --- a/lokingyql/contrib/table/binder.py +++ b/lokingyql/contrib/table/binder.py @@ -25,7 +25,7 @@ def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[] # Adding inputs passed as parameters if inputs: - self.inputs = [ self.addInput(key.etree) for key in inputs ] + self.inputs = [ self.addInput(key) for key in inputs ] # Adding paging if paging: From 45ec648459f66727ab7f207be604542c96cbd135 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 5 Apr 2015 01:32:07 +0200 Subject: [PATCH 119/582] renaming file --- lokingyql/contrib/table/{run_test.sh => run_tests.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lokingyql/contrib/table/{run_test.sh => run_tests.sh} (100%) diff --git a/lokingyql/contrib/table/run_test.sh b/lokingyql/contrib/table/run_tests.sh similarity index 100% rename from lokingyql/contrib/table/run_test.sh rename to lokingyql/contrib/table/run_tests.sh From 122b9f0161e0838cb314419d4b95464c1da3f87b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 5 Apr 2015 01:32:46 +0200 Subject: [PATCH 120/582] logging added --- lokingyql/contrib/table/tests.py | 49 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/lokingyql/contrib/table/tests.py b/lokingyql/contrib/table/tests.py index c62038b..34a1264 100755 --- a/lokingyql/contrib/table/tests.py +++ b/lokingyql/contrib/table/tests.py @@ -1,4 +1,4 @@ -import os +import os, logging import pdb import unittest from xml.dom import minidom @@ -9,6 +9,8 @@ import readline, rlcompleter readline.parse_and_bind('tab: complete') +logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(funcName)s] %(message)s \n") + class TestTable(unittest.TestCase): def setUp(self,): @@ -46,7 +48,7 @@ def setUp(self,): self.paging = BinderPage('page', start, pageSize, total) def xml_pretty_print(self, data): - """Pretty print xml data + """Pretty logging.debug xml data """ raw_string = xtree.tostring(data, 'utf-8') parsed_string = minidom.parseString(raw_string) @@ -54,7 +56,7 @@ def xml_pretty_print(self, data): def test_add_binder(self,): self.assertEqual(self.table.addBinder(self.binder),True) - print self.xml_pretty_print(self.table.etree) + logging.debug(self.xml_pretty_print(self.table.etree)) def test_remove_binder(self,): self.binder.addInput(self.key) @@ -71,52 +73,51 @@ def test_remove_binder(self,): def test_add_input_to_binder(self,): self.assertEqual(self.binder.addInput(self.key),True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_remove_input(self,): self.assertEqual(self.binder.addInput(self.key),True) self.assertEqual(self.binder.addInput(self.key2),True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.removeInput(key_id='artist'),True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_add_function_from_file(self,): self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_remove_function(self,): self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEqual(self.binder.removeFunction(), True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_add_paging(self,): - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.addPaging(self.paging), True) - print(self.binder.paging) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_remove_paging(self,): - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.addPaging(self.paging), True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEqual(self.binder.removePaging(), True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_add_url(self,): url = 'http://josuebrunel.org/service.js' self.assertEquals(self.binder.addUrl(url), True) - print(self.binder.urls) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.binder.urls) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_remove_url(self,): url = 'http://josuebrunel.org/service.js' url2 = 'http://google.com' self.assertEquals(self.binder.addUrl(url), True) self.assertEquals(self.binder.addUrl(url2), True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.removeUrl(url), True) - print self.xml_pretty_print(self.binder.etree) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_save_file(self,): self.table.save() @@ -160,16 +161,16 @@ def test_create_table_with_binder(self,): self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) def test_add_function_table(self): - print(self.xml_pretty_print(self.table.etree)) + logging.debug(self.xml_pretty_print(self.table.etree)) self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) - print(self.xml_pretty_print(self.table.etree)) + logging.debug(self.xml_pretty_print(self.table.etree)) def test_remove_function_table(self,): - print(self.xml_pretty_print(self.table.etree)) + logging.debug(self.xml_pretty_print(self.table.etree)) self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) - print(self.xml_pretty_print(self.table.etree)) + logging.debug(self.xml_pretty_print(self.table.etree)) self.assertEquals(self.table.removeFunction(),True) - print(self.xml_pretty_print(self.table.etree)) + logging.debug(self.xml_pretty_print(self.table.etree)) def tearUp(self): os.path.unlink('tests_data/mytest.xml') From b5c1157bb94c3d1e5d87cc51bbf35e156a5bab45 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 5 Apr 2015 12:35:42 +0200 Subject: [PATCH 121/582] fix #40 : that slolution kind of worked --- lokingyql/contrib/table/binder.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lokingyql/contrib/table/binder.py b/lokingyql/contrib/table/binder.py index 89d35cb..a024c30 100755 --- a/lokingyql/contrib/table/binder.py +++ b/lokingyql/contrib/table/binder.py @@ -25,7 +25,7 @@ def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[] # Adding inputs passed as parameters if inputs: - self.inputs = [ self.addInput(key) for key in inputs ] + [ self.addInput(key) for key in inputs ] # Adding paging if paging: @@ -48,7 +48,9 @@ def _buildElementTree(self,): def addUrl(self, url): """Adds url to binder """ - self.urls.append(url) + if not url in self.urls: + self.urls.append(url) + root = self.etree t_urls = root.find('urls') @@ -70,6 +72,8 @@ def removeUrl(self, url): for t_url in t_urls.findall('url'): if t_url.text == url.strip(): t_urls.remove(t_url) + if url in self.urls: + self.urls.remove(url) return True return False @@ -77,7 +81,8 @@ def removeUrl(self, url): def addInput(self, key): """Add key element to the binder """ - self.inputs.append(key) + if not key in self.inputs: + self.inputs.append(key) root = self.etree t_input = root.find('inputs') From 5232b7683977e4cc58b194000317f1b797e9ca51 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 5 Apr 2015 15:40:04 +0200 Subject: [PATCH 122/582] test_create_table_with_two_binders added and Ok --- lokingyql/contrib/table/tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lokingyql/contrib/table/tests.py b/lokingyql/contrib/table/tests.py index 34a1264..40fcaea 100755 --- a/lokingyql/contrib/table/tests.py +++ b/lokingyql/contrib/table/tests.py @@ -160,6 +160,15 @@ def test_create_table_with_binder(self,): table.save(name='mytable', path='tests_data') self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + def test_create_table_with_two_binders(self,): + self.binder.addInput(self.key) + self.binder.addFunction('', from_file='tests_data/jscode.js') + self.table_desc['bindings'] = [self.binder, self.binder_insert] + table = Table(**self.table_desc) + table.save(name='mytable', path='tests_data') + logging.debug(self.xml_pretty_print(table.etree)) + self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + def test_add_function_table(self): logging.debug(self.xml_pretty_print(self.table.etree)) self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) From 71570b80f7a8397467f86dc2a09914c1f3986649 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 5 Apr 2015 15:50:43 +0200 Subject: [PATCH 123/582] better logger in tests.py --- lokingyql/contrib/table/tests.py | 22 +++++++++++++++++++--- run_tests.sh | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lokingyql/contrib/table/tests.py b/lokingyql/contrib/table/tests.py index 40fcaea..e06f712 100755 --- a/lokingyql/contrib/table/tests.py +++ b/lokingyql/contrib/table/tests.py @@ -9,7 +9,8 @@ import readline, rlcompleter readline.parse_and_bind('tab: complete') -logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(funcName)s] %(message)s \n") +logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") +logging.getLogger(__name__) class TestTable(unittest.TestCase): @@ -72,21 +73,26 @@ def test_remove_binder(self,): self.assertEqual(os.path.isfile('tests_data/after.xml'),True) def test_add_input_to_binder(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEqual(self.binder.addInput(self.key),True) logging.debug(self.xml_pretty_print(self.binder.etree)) def test_remove_input(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEqual(self.binder.addInput(self.key),True) + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEqual(self.binder.addInput(self.key2),True) logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.removeInput(key_id='artist'),True) logging.debug(self.xml_pretty_print(self.binder.etree)) def test_add_function_from_file(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) logging.debug(self.xml_pretty_print(self.binder.etree)) def test_remove_function(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEqual(self.binder.removeFunction(), True) @@ -106,14 +112,16 @@ def test_remove_paging(self,): def test_add_url(self,): url = 'http://josuebrunel.org/service.js' + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.addUrl(url), True) - logging.debug(self.binder.urls) logging.debug(self.xml_pretty_print(self.binder.etree)) def test_remove_url(self,): url = 'http://josuebrunel.org/service.js' url2 = 'http://google.com' + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.addUrl(url), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.addUrl(url2), True) logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.removeUrl(url), True) @@ -138,36 +146,44 @@ def test_save_to_different_location(self,): def test_create_table(self,): self.binder.addInput(self.key) self.binder.addFunction('', from_file='tests_data/jscode.js') + logging.debug(self.xml_pretty_print(self.table.etree)) self.table.addBinder(self.binder) self.table.save(name='mytable', path='tests_data') self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + logging.debug(self.xml_pretty_print(self.table.etree)) def test_create_table_and_add_two_binders(self,): self.binder.addInput(self.key) self.binder_insert.addInput(self.key) self.binder.addFunction('', from_file='tests_data/jscode.js') self.binder_insert.addFunction("console.log('hello this is an insert function'); ") + logging.debug(self.xml_pretty_print(self.table.etree)) self.table.addBinder(self.binder) + logging.debug(self.xml_pretty_print(self.table.etree)) self.table.addBinder(self.binder_insert) self.table.save(name='mytable', path='tests_data') self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + logging.debug(self.xml_pretty_print(self.table.etree)) def test_create_table_with_binder(self,): self.binder.addInput(self.key) self.binder.addFunction('', from_file='tests_data/jscode.js') self.table_desc['bindings'] = [self.binder] table = Table(**self.table_desc) + logging.debug(self.xml_pretty_print(table.etree)) table.save(name='mytable', path='tests_data') self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + logging.debug(self.xml_pretty_print(table.etree)) def test_create_table_with_two_binders(self,): self.binder.addInput(self.key) self.binder.addFunction('', from_file='tests_data/jscode.js') self.table_desc['bindings'] = [self.binder, self.binder_insert] table = Table(**self.table_desc) - table.save(name='mytable', path='tests_data') logging.debug(self.xml_pretty_print(table.etree)) + table.save(name='mytable', path='tests_data') self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + logging.debug(self.xml_pretty_print(table.etree)) def test_add_function_table(self): logging.debug(self.xml_pretty_print(self.table.etree)) diff --git a/run_tests.sh b/run_tests.sh index bde282e..38c28c8 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,7 +4,7 @@ # Filename : run_tests.sh # Description : # Creation Date : 03-03-2015 -# Last Modified : Tue 03 Mar 2015 07:03:37 AM CST +# Last Modified : Sun 05 Apr 2015 01:46:24 AM CEST # ################################################## From 9c4c1e5adcbe7d773b3943b9c7fb4c6d0508872e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 04:52:05 +0200 Subject: [PATCH 124/582] renaming the whole thing --- README.md | 16 ++++++++-------- lokingyql/__init__.py | 3 --- myql/__init__.py | 3 +++ {lokingyql => myql}/contrib/__init__.py | 0 {lokingyql => myql}/contrib/table/__init__.py | 0 {lokingyql => myql}/contrib/table/binder.py | 0 {lokingyql => myql}/contrib/table/run_tests.sh | 0 {lokingyql => myql}/contrib/table/table.py | 0 {lokingyql => myql}/contrib/table/tests.py | 0 .../contrib/table/tests_data/jscode.js | 0 .../contrib/yahooauth/__init__.py | 0 {lokingyql => myql}/contrib/yahooauth/readme.md | 0 .../contrib/yahooauth/yahooauth.py | 0 {lokingyql => myql}/errors.py | 0 lokingyql/lokingyql.py => myql/myql.py | 2 +- setup.py | 2 +- tests/tests.py | 8 ++++---- 17 files changed, 17 insertions(+), 17 deletions(-) delete mode 100755 lokingyql/__init__.py create mode 100755 myql/__init__.py rename {lokingyql => myql}/contrib/__init__.py (100%) rename {lokingyql => myql}/contrib/table/__init__.py (100%) rename {lokingyql => myql}/contrib/table/binder.py (100%) rename {lokingyql => myql}/contrib/table/run_tests.sh (100%) rename {lokingyql => myql}/contrib/table/table.py (100%) rename {lokingyql => myql}/contrib/table/tests.py (100%) rename {lokingyql => myql}/contrib/table/tests_data/jscode.js (100%) rename {lokingyql => myql}/contrib/yahooauth/__init__.py (100%) rename {lokingyql => myql}/contrib/yahooauth/readme.md (100%) rename {lokingyql => myql}/contrib/yahooauth/yahooauth.py (100%) rename {lokingyql => myql}/errors.py (100%) rename lokingyql/lokingyql.py => myql/myql.py (99%) diff --git a/README.md b/README.md index a7ac0ef..4207c0e 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ lokingYQL ========= -LokingYQL is a Python wrapper of the Yahoo Query Language. +MYQL is a Python wrapper of the Yahoo Query Language. Yahoo! Query Langauge Documentation and Support =============================================== @@ -42,8 +42,8 @@ how to use ========== ```python ->>> import lokingyql ->>> yql = lokingyql.LokingYQL() +>>> import myql +>>> yql = myql.MYQL() >>> yql.diagnostics = True # To turn diagnostics on ``` @@ -51,7 +51,7 @@ access to community tables -------------------------- ```python ->>> yql = lokingyql.LokingYQL() +>>> yql = myql.MYQL() >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') >>> rep.json() {u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} @@ -65,8 +65,8 @@ access to community tables ***OR*** ```python ->>> import lokingyql ->>> yql = lokingyql.LokingYQL(community=True) +>>> import myql +>>> yql = myql.MYQL(community=True) >>> # do your magic ``` @@ -76,8 +76,8 @@ changing response format (xml or json) The response format is by default ***json***. ```python ->>> import lokingyql ->>> yql = lokingyql.LokingYQL(format='xml', community=True) +>>> import myql +>>> yql = myql.MYQL(format='xml', community=True) >>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') >>> rep.text u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' diff --git a/lokingyql/__init__.py b/lokingyql/__init__.py deleted file mode 100755 index 077cb64..0000000 --- a/lokingyql/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import contrib -import errors -from lokingyql import LokingYQL diff --git a/myql/__init__.py b/myql/__init__.py new file mode 100755 index 0000000..a1be23e --- /dev/null +++ b/myql/__init__.py @@ -0,0 +1,3 @@ +import contrib +import errors +from myql import MYQL diff --git a/lokingyql/contrib/__init__.py b/myql/contrib/__init__.py similarity index 100% rename from lokingyql/contrib/__init__.py rename to myql/contrib/__init__.py diff --git a/lokingyql/contrib/table/__init__.py b/myql/contrib/table/__init__.py similarity index 100% rename from lokingyql/contrib/table/__init__.py rename to myql/contrib/table/__init__.py diff --git a/lokingyql/contrib/table/binder.py b/myql/contrib/table/binder.py similarity index 100% rename from lokingyql/contrib/table/binder.py rename to myql/contrib/table/binder.py diff --git a/lokingyql/contrib/table/run_tests.sh b/myql/contrib/table/run_tests.sh similarity index 100% rename from lokingyql/contrib/table/run_tests.sh rename to myql/contrib/table/run_tests.sh diff --git a/lokingyql/contrib/table/table.py b/myql/contrib/table/table.py similarity index 100% rename from lokingyql/contrib/table/table.py rename to myql/contrib/table/table.py diff --git a/lokingyql/contrib/table/tests.py b/myql/contrib/table/tests.py similarity index 100% rename from lokingyql/contrib/table/tests.py rename to myql/contrib/table/tests.py diff --git a/lokingyql/contrib/table/tests_data/jscode.js b/myql/contrib/table/tests_data/jscode.js similarity index 100% rename from lokingyql/contrib/table/tests_data/jscode.js rename to myql/contrib/table/tests_data/jscode.js diff --git a/lokingyql/contrib/yahooauth/__init__.py b/myql/contrib/yahooauth/__init__.py similarity index 100% rename from lokingyql/contrib/yahooauth/__init__.py rename to myql/contrib/yahooauth/__init__.py diff --git a/lokingyql/contrib/yahooauth/readme.md b/myql/contrib/yahooauth/readme.md similarity index 100% rename from lokingyql/contrib/yahooauth/readme.md rename to myql/contrib/yahooauth/readme.md diff --git a/lokingyql/contrib/yahooauth/yahooauth.py b/myql/contrib/yahooauth/yahooauth.py similarity index 100% rename from lokingyql/contrib/yahooauth/yahooauth.py rename to myql/contrib/yahooauth/yahooauth.py diff --git a/lokingyql/errors.py b/myql/errors.py similarity index 100% rename from lokingyql/errors.py rename to myql/errors.py diff --git a/lokingyql/lokingyql.py b/myql/myql.py similarity index 99% rename from lokingyql/lokingyql.py rename to myql/myql.py index 85a38b9..ff41911 100755 --- a/lokingyql/lokingyql.py +++ b/myql/myql.py @@ -7,7 +7,7 @@ __author__ = 'Josue Kouka' __email__ = 'josuebrunel@gmail.com' -class LokingYQL(object): +class MYQL(object): '''Yet another Python Yahoo! Query Language Wrapper Attributes: - url : data provider url diff --git a/setup.py b/setup.py index 350b4f3..e307827 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ required = f.read().splitlines() setup( - name = "lokingyql", + name = "myql", version = "0.5", description = "Python Wrapper for the Yahoo ! Query Language", long_description = "", diff --git a/tests/tests.py b/tests/tests.py index 1497e2b..4723e63 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,15 +1,15 @@ import unittest -from lokingyql import LokingYQL -from lokingyql.errors import NoTableSelectedError -#from lokingyql.contrib import YahooOAuth +from myql import MYQL +from myql.errors import NoTableSelectedError +#from myql.contrib import YahooOAuth from test_config import consumer_key, consumer_secret class LokingYqlTest(unittest.TestCase): def setUp(self,): - self.yql = LokingYQL() + self.yql = MYQL() #self.yoauth = YahooOAuth(consumer_key, consumer_secret) def tearUp(self,): From ccbbfe0c9bd609b9736a4f5e0860268f5d58ff65 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 04:54:39 +0200 Subject: [PATCH 125/582] changing the tile --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4207c0e..b715812 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -lokingYQL +MYQL ========= MYQL is a Python wrapper of the Yahoo Query Language. From 99420d589c8794f9e0c05d6bdad42aef030c2c00 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 05:56:59 +0200 Subject: [PATCH 126/582] #41: create_binder_with_list_urls test added --- myql/contrib/table/tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/myql/contrib/table/tests.py b/myql/contrib/table/tests.py index e06f712..24373a8 100755 --- a/myql/contrib/table/tests.py +++ b/myql/contrib/table/tests.py @@ -110,6 +110,16 @@ def test_remove_paging(self,): self.assertEqual(self.binder.removePaging(), True) logging.debug(self.xml_pretty_print(self.binder.etree)) + def test_create_binder_with_urls(self,): + url = 'http://josuebrunel.org/service/v1' + url2 = 'http://josuebrunel.org/service/v1/?name=lol' + self.binder_desc['urls'] = [url, url2] + binder = Binder(**self.binder_desc) + logging.debug(self.xml_pretty_print(binder.etree)) + #self.assertEquals(self.binder.addUrl(url), True) + #logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_add_url(self,): url = 'http://josuebrunel.org/service.js' logging.debug(self.xml_pretty_print(self.binder.etree)) From 9d961b7ca001398274cc001bd543a151f526bdcd Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 05:57:51 +0200 Subject: [PATCH 127/582] fix #41: it works now --- myql/contrib/table/binder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index a024c30..f29afbc 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -19,9 +19,13 @@ def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[] self.urls = urls self.inputs = inputs self.paging = paging + # Builds the element tree self.etree = self._buildElementTree() + # Adding urls + if urls: + [ self.addUrl(url) for url in urls ] # Adding inputs passed as parameters if inputs: From 09a2029a6299a0e297c3675ee14cb2c29bbd3443 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 07:01:35 +0200 Subject: [PATCH 128/582] #36 : Binder Meta class added --- myql/contrib/table/binder.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index f29afbc..04656f3 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -231,3 +231,23 @@ def __buildElementTree(self,): return t_paging +class BinderMeta(type): + + BINDER_KEY = ['name', 'itemPath', 'produces', 'pollingFrequencySeconds', 'urls', 'keys', 'pages'] + + def __new__(cls, name, bases, dct): + if name != 'BinderModel': + binder_attr = {key: value for (key, value) in dct.items() if key in cls.BINDER_KEY} + binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BinderKey)] + dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} + dct['binder'] = Binder(**binder_attr) + # Add KeyException Management + return super(BinderMeta,cls).__new__(cls, name, (Binder,), dct) + + def toxml(cls,): + return ctree.tostring(cls.binder.etree) + + +class BinderModel(Binder): + __metaclass__ = BinderMeta + From 6b28d504d0adb35dac66a262c4f5bd31a8914c4f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 07:05:16 +0200 Subject: [PATCH 129/582] #36 : table metaclass added --- myql/contrib/table/table.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index 14247d6..a98d46b 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -158,3 +158,24 @@ def removeFunction(self): print(e) return False + + +class TableMeta(type): + + TABLE_KEYS = ['name', 'author', 'apiKeyURL', 'documentationURL', 'sampleQuery'] + + def __new__(cls, name, bases, dct): + if name != 'TableModel': + table_attr = {key: value for (key, value) in dct.items() if key in cls.TABLE_KEYS } + table_attr['bindings'] = [ value.binder for value in dct.values() if hasattr(value, 'binder') and isinstance(value, BinderMeta) ] + dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} + print dct + + return super(TableMeta, cls).__new__(cls, name, (Table,), dct) + + def toxml(cls,): + return ctree.tostring(cls.table.etree) + +class TableModel(Table): + __metaclass__ = TableMeta + From 3620cfb4fc2a97a2c4d45ba029b084227533a8a1 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 07:06:17 +0200 Subject: [PATCH 130/582] fix ctree/xtree and unexpeted indent --- myql/contrib/table/binder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 04656f3..63aca03 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -245,7 +245,7 @@ def __new__(cls, name, bases, dct): return super(BinderMeta,cls).__new__(cls, name, (Binder,), dct) def toxml(cls,): - return ctree.tostring(cls.binder.etree) + return xtree.tostring(cls.binder.etree) class BinderModel(Binder): From 376582fe63ba033c99d10f9ceab0ba61fa88ffb7 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 07:06:56 +0200 Subject: [PATCH 131/582] fix ctree/xtree --- myql/contrib/table/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index a98d46b..2afd5b6 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -174,7 +174,7 @@ def __new__(cls, name, bases, dct): return super(TableMeta, cls).__new__(cls, name, (Table,), dct) def toxml(cls,): - return ctree.tostring(cls.table.etree) + return xtree.tostring(cls.table.etree) class TableModel(Table): __metaclass__ = TableMeta From 6f8da00c3a4e6e6ab0aa8a79f74cc58febd297ee Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 11:30:50 +0200 Subject: [PATCH 132/582] #42 : test_create_binder_with_paging added --- myql/contrib/table/tests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/myql/contrib/table/tests.py b/myql/contrib/table/tests.py index 24373a8..199eba0 100755 --- a/myql/contrib/table/tests.py +++ b/myql/contrib/table/tests.py @@ -103,6 +103,18 @@ def test_add_paging(self,): self.assertEquals(self.binder.addPaging(self.paging), True) logging.debug(self.xml_pretty_print(self.binder.etree)) + def test_create_table_with_paging(self,): + start= {'id': 'ItemPage', 'default': '1'} + pageSize= {'id':'Count' ,'max':'25'} + total= {'default': '10'} + paging = BinderPage('page', start, pageSize, total) + logging.debug(self.xml_pretty_print(paging.etree)) + self.binder_desc['paging']=paging + logging.debug(self.binder_desc) + binder = Binder(**self.binder_desc) + self.assertNotEqual(binder.paging,None) + logging.debug(self.xml_pretty_print(binder.etree)) + def test_remove_paging(self,): logging.debug(self.xml_pretty_print(self.binder.etree)) self.assertEquals(self.binder.addPaging(self.paging), True) From 418a6ddacd9cf573fba45a3eb250ee149a7367c2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 11:32:05 +0200 Subject: [PATCH 133/582] fix #42: create binder with paging ok --- myql/contrib/table/binder.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 63aca03..0734d13 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -151,9 +151,10 @@ def removeFunction(self,): def addPaging(self, paging): """Adds paging to binder """ - self.paging = paging - root = self.etree + if not self.paging: + self.paging = paging + root = self.etree try: root.append(paging.etree) return True @@ -221,13 +222,13 @@ def __buildElementTree(self,): """Turns object into into an ElementTree """ t_paging = xtree.Element('paging') - + t_paging.set('model',self.model) for key in self.__dict__.keys(): if key != 'model': t_tag = xtree.SubElement(t_paging, key) for item in self.__dict__[key].items() : t_tag.set(*item) - + return t_paging From f3f703e311bf9b71539c5b7af9538727744573d8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 6 Apr 2015 11:47:54 +0200 Subject: [PATCH 134/582] fix #36: using metaclass OK ^_^ --- myql/contrib/table/binder.py | 3 ++- myql/contrib/table/table.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 0734d13..ab7305d 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -237,13 +237,14 @@ class BinderMeta(type): BINDER_KEY = ['name', 'itemPath', 'produces', 'pollingFrequencySeconds', 'urls', 'keys', 'pages'] def __new__(cls, name, bases, dct): + if name != 'BinderModel': binder_attr = {key: value for (key, value) in dct.items() if key in cls.BINDER_KEY} binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BinderKey)] dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} dct['binder'] = Binder(**binder_attr) # Add KeyException Management - return super(BinderMeta,cls).__new__(cls, name, (Binder,), dct) + return super(BinderMeta,cls).__new__(cls, name, (Binder,), dct) def toxml(cls,): return xtree.tostring(cls.binder.etree) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index 2afd5b6..a786fa5 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -2,6 +2,8 @@ from xml.dom import minidom from xml.etree import cElementTree as xtree +from binder import BinderMeta + class Table(object): """Class representating a YQL Table """ @@ -169,6 +171,7 @@ def __new__(cls, name, bases, dct): table_attr = {key: value for (key, value) in dct.items() if key in cls.TABLE_KEYS } table_attr['bindings'] = [ value.binder for value in dct.values() if hasattr(value, 'binder') and isinstance(value, BinderMeta) ] dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} + dct['table'] = Table(**table_attr) print dct return super(TableMeta, cls).__new__(cls, name, (Table,), dct) From dc7be4fefac82028cc9f8e1e31375e8e592e2684 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 7 Apr 2015 12:08:58 +0200 Subject: [PATCH 135/582] link and version modified in setup --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e307827..a3931c5 100755 --- a/setup.py +++ b/setup.py @@ -6,12 +6,12 @@ setup( name = "myql", - version = "0.5", + version = "1.2", description = "Python Wrapper for the Yahoo ! Query Language", long_description = "", author = "Josue Kouka", author_email = "josuebrunel@gmail.com", - url = "https://github.com/josuebrunel/lokingYQL", + url = "https://github.com/josuebrunel/MYQL", packages = find_packages(), platforms=['Any'], license='BSD', From a9e2e56cf0ebf193526f7bdabcc39d3eb2beb566 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 7 Apr 2015 13:08:17 +0200 Subject: [PATCH 136/582] fix #47: its works now --- myql/contrib/table/binder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index ab7305d..b5a8898 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -241,6 +241,7 @@ def __new__(cls, name, bases, dct): if name != 'BinderModel': binder_attr = {key: value for (key, value) in dct.items() if key in cls.BINDER_KEY} binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BinderKey)] + binder_attr['paging'] = [ value for value in dct.values() if isinstance(value, BinderPage)][0] dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} dct['binder'] = Binder(**binder_attr) # Add KeyException Management From 00e8dfaa9e0df03971c23de8dd198fcb35a63eb0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 7 Apr 2015 13:08:17 +0200 Subject: [PATCH 137/582] fix #43: its works now --- myql/contrib/table/binder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index ab7305d..b5a8898 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -241,6 +241,7 @@ def __new__(cls, name, bases, dct): if name != 'BinderModel': binder_attr = {key: value for (key, value) in dct.items() if key in cls.BINDER_KEY} binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BinderKey)] + binder_attr['paging'] = [ value for value in dct.values() if isinstance(value, BinderPage)][0] dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} dct['binder'] = Binder(**binder_attr) # Add KeyException Management From f5ad906d107b2e7d37b0be9fc68c65195188c0d2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 7 Apr 2015 14:12:36 +0200 Subject: [PATCH 138/582] fix #43 : You can still suck Josh :+1: --- myql/contrib/table/binder.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index b5a8898..b4ccc2b 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -241,7 +241,10 @@ def __new__(cls, name, bases, dct): if name != 'BinderModel': binder_attr = {key: value for (key, value) in dct.items() if key in cls.BINDER_KEY} binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BinderKey)] - binder_attr['paging'] = [ value for value in dct.values() if isinstance(value, BinderPage)][0] + paging = [ value for value in dct.values() if isinstance(value, BinderPage)] + if paging : + binder_attr['paging'] = paging[0] + dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} dct['binder'] = Binder(**binder_attr) # Add KeyException Management From a7bb522b02adab46e2496c287fe3697cd02a4e1a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Apr 2015 15:23:20 +0200 Subject: [PATCH 139/582] refactorint tests --- myql/contrib/table/run_tests.sh | 7 - myql/contrib/table/table.py | 1 - myql/contrib/table/tests.py | 228 -------------------- myql/contrib/table/tests_data/jscode.js | 5 - run_tests.sh | 18 +- tests/__init__.py | 2 +- tests/tests.py | 274 +++++++++++++++++++----- 7 files changed, 232 insertions(+), 303 deletions(-) delete mode 100755 myql/contrib/table/run_tests.sh delete mode 100755 myql/contrib/table/tests.py delete mode 100755 myql/contrib/table/tests_data/jscode.js diff --git a/myql/contrib/table/run_tests.sh b/myql/contrib/table/run_tests.sh deleted file mode 100755 index 33883f0..0000000 --- a/myql/contrib/table/run_tests.sh +++ /dev/null @@ -1,7 +0,0 @@ -if [ ! -z $1 ]; then - method=".$1" -else - method='' -fi - -python -m unittest tests.TestTable$method diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index a786fa5..e4c3846 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -172,7 +172,6 @@ def __new__(cls, name, bases, dct): table_attr['bindings'] = [ value.binder for value in dct.values() if hasattr(value, 'binder') and isinstance(value, BinderMeta) ] dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} dct['table'] = Table(**table_attr) - print dct return super(TableMeta, cls).__new__(cls, name, (Table,), dct) diff --git a/myql/contrib/table/tests.py b/myql/contrib/table/tests.py deleted file mode 100755 index 199eba0..0000000 --- a/myql/contrib/table/tests.py +++ /dev/null @@ -1,228 +0,0 @@ -import os, logging -import pdb -import unittest -from xml.dom import minidom -from xml.etree import cElementTree as xtree -from binder import Binder, BinderKey, BinderPage -from table import Table - -import readline, rlcompleter -readline.parse_and_bind('tab: complete') - -logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") -logging.getLogger(__name__) - -class TestTable(unittest.TestCase): - - def setUp(self,): - self.table_desc = { - 'name': 'mytest', - 'author': 'josuebrunel', - 'apiKeyURL': 'http://josuebrunel.org/api', - 'documentationURL': 'http://josuebrunel.org/doc.html', - 'sampleQuery': 'SELECT * FROM mytable', - } - - self.table = Table(**self.table_desc) - - self.binder_desc = { - 'name': 'select', - 'itemPath': 'products.product', - 'produces': 'xml' - } - - self.binder = Binder(**self.binder_desc) - self.binder_insert = Binder('insert','products.product','json') - - self.key_desc = { - 'id': 'artist', - 'type': 'xs:string', - 'paramType': 'path' - } - - self.key = BinderKey(**self.key_desc) - self.key2 = BinderKey(id='song', type='xs:string', paramType='path', required='true') - - start= {'id': 'ItemPage', 'default': '1'} - pageSize= {'id':'Count' ,'max':'25'} - total= {'default': '10'} - self.paging = BinderPage('page', start, pageSize, total) - - def xml_pretty_print(self, data): - """Pretty logging.debug xml data - """ - raw_string = xtree.tostring(data, 'utf-8') - parsed_string = minidom.parseString(raw_string) - return parsed_string.toprettyxml(indent='\t') - - def test_add_binder(self,): - self.assertEqual(self.table.addBinder(self.binder),True) - logging.debug(self.xml_pretty_print(self.table.etree)) - - def test_remove_binder(self,): - self.binder.addInput(self.key) - self.binder_insert.addInput(self.key) - self.binder.addFunction('', from_file='tests_data/jscode.js') - self.binder_insert.addFunction("console.log('hello this is an insert function'); ") - self.table.addBinder(self.binder) - self.table.addBinder(self.binder_insert) - self.table.save(name='before', path='tests_data') - self.assertEqual(os.path.isfile('tests_data/before.xml'),True) - self.table.removeBinder('select') - self.table.save(name='after', path='tests_data') - self.assertEqual(os.path.isfile('tests_data/after.xml'),True) - - def test_add_input_to_binder(self,): - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEqual(self.binder.addInput(self.key),True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - - def test_remove_input(self,): - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEqual(self.binder.addInput(self.key),True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEqual(self.binder.addInput(self.key2),True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEquals(self.binder.removeInput(key_id='artist'),True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - - def test_add_function_from_file(self,): - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - - def test_remove_function(self,): - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEqual(self.binder.removeFunction(), True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - - def test_add_paging(self,): - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEquals(self.binder.addPaging(self.paging), True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - - def test_create_table_with_paging(self,): - start= {'id': 'ItemPage', 'default': '1'} - pageSize= {'id':'Count' ,'max':'25'} - total= {'default': '10'} - paging = BinderPage('page', start, pageSize, total) - logging.debug(self.xml_pretty_print(paging.etree)) - self.binder_desc['paging']=paging - logging.debug(self.binder_desc) - binder = Binder(**self.binder_desc) - self.assertNotEqual(binder.paging,None) - logging.debug(self.xml_pretty_print(binder.etree)) - - def test_remove_paging(self,): - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEquals(self.binder.addPaging(self.paging), True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEqual(self.binder.removePaging(), True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - - def test_create_binder_with_urls(self,): - url = 'http://josuebrunel.org/service/v1' - url2 = 'http://josuebrunel.org/service/v1/?name=lol' - self.binder_desc['urls'] = [url, url2] - binder = Binder(**self.binder_desc) - logging.debug(self.xml_pretty_print(binder.etree)) - #self.assertEquals(self.binder.addUrl(url), True) - #logging.debug(self.xml_pretty_print(self.binder.etree)) - - - def test_add_url(self,): - url = 'http://josuebrunel.org/service.js' - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEquals(self.binder.addUrl(url), True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - - def test_remove_url(self,): - url = 'http://josuebrunel.org/service.js' - url2 = 'http://google.com' - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEquals(self.binder.addUrl(url), True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEquals(self.binder.addUrl(url2), True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - self.assertEquals(self.binder.removeUrl(url), True) - logging.debug(self.xml_pretty_print(self.binder.etree)) - - def test_save_file(self,): - self.table.save() - self.assertEquals(os.path.isfile('mytest.xml'),True) - - def test_save_with_another_name(self): - name = "tests_data/toto" - self.table.save(name) - self.assertEquals(os.path.isfile(name+'.xml'),True) - - def test_save_to_different_location(self,): - fname = "titi" - path = 'tests_data' - name = os.path.join(path,fname) - self.table.save(name=fname, path=path) - self.assertEquals(os.path.isfile(name+'.xml'),True) - - def test_create_table(self,): - self.binder.addInput(self.key) - self.binder.addFunction('', from_file='tests_data/jscode.js') - logging.debug(self.xml_pretty_print(self.table.etree)) - self.table.addBinder(self.binder) - self.table.save(name='mytable', path='tests_data') - self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) - logging.debug(self.xml_pretty_print(self.table.etree)) - - def test_create_table_and_add_two_binders(self,): - self.binder.addInput(self.key) - self.binder_insert.addInput(self.key) - self.binder.addFunction('', from_file='tests_data/jscode.js') - self.binder_insert.addFunction("console.log('hello this is an insert function'); ") - logging.debug(self.xml_pretty_print(self.table.etree)) - self.table.addBinder(self.binder) - logging.debug(self.xml_pretty_print(self.table.etree)) - self.table.addBinder(self.binder_insert) - self.table.save(name='mytable', path='tests_data') - self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) - logging.debug(self.xml_pretty_print(self.table.etree)) - - def test_create_table_with_binder(self,): - self.binder.addInput(self.key) - self.binder.addFunction('', from_file='tests_data/jscode.js') - self.table_desc['bindings'] = [self.binder] - table = Table(**self.table_desc) - logging.debug(self.xml_pretty_print(table.etree)) - table.save(name='mytable', path='tests_data') - self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) - logging.debug(self.xml_pretty_print(table.etree)) - - def test_create_table_with_two_binders(self,): - self.binder.addInput(self.key) - self.binder.addFunction('', from_file='tests_data/jscode.js') - self.table_desc['bindings'] = [self.binder, self.binder_insert] - table = Table(**self.table_desc) - logging.debug(self.xml_pretty_print(table.etree)) - table.save(name='mytable', path='tests_data') - self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) - logging.debug(self.xml_pretty_print(table.etree)) - - def test_add_function_table(self): - logging.debug(self.xml_pretty_print(self.table.etree)) - self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) - logging.debug(self.xml_pretty_print(self.table.etree)) - - def test_remove_function_table(self,): - logging.debug(self.xml_pretty_print(self.table.etree)) - self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) - logging.debug(self.xml_pretty_print(self.table.etree)) - self.assertEquals(self.table.removeFunction(),True) - logging.debug(self.xml_pretty_print(self.table.etree)) - - def tearUp(self): - os.path.unlink('tests_data/mytest.xml') - os.path.unlink('tests_data/toto.xml') - -if '__main__' == __name__: - unittest.main() - diff --git a/myql/contrib/table/tests_data/jscode.js b/myql/contrib/table/tests_data/jscode.js deleted file mode 100755 index f3b503b..0000000 --- a/myql/contrib/table/tests_data/jscode.js +++ /dev/null @@ -1,5 +0,0 @@ - // Include the flickr signing library - y.include("http:blog.pipes.yahoo.net/wp-content/uploads/flickr.js"); - // GET the flickr result using a signed url - var fs = new flickrSigner(api_key,secret); - response.object = y.rest(fs.createUrl({method:method, format:""})).get().response(); diff --git a/run_tests.sh b/run_tests.sh index 38c28c8..33883f0 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,15 +1,7 @@ -################################################## -# -# Author : yosuke -# Filename : run_tests.sh -# Description : -# Creation Date : 03-03-2015 -# Last Modified : Sun 05 Apr 2015 01:46:24 AM CEST -# -################################################## - -if [ ! -z $1 ]; then - test=".$1" +if [ ! -z $1 ]; then + method=".$1" +else + method='' fi -python -m unittest "tests.LokingYqlTest${test}" +python -m unittest tests.TestTable$method diff --git a/tests/__init__.py b/tests/__init__.py index e92dd39..031965b 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1 @@ -from tests import LokingYqlTest +from tests import TestTable diff --git a/tests/tests.py b/tests/tests.py index 4723e63..99a9aa2 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,50 +1,228 @@ +import os, logging +import pdb import unittest +from xml.dom import minidom +from xml.etree import cElementTree as xtree +from myql.contrib.table import Binder, BinderKey, BinderPage +from myql.contrib.table import Table + +import readline, rlcompleter +readline.parse_and_bind('tab: complete') + +logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") +logging.getLogger(__name__) + +class TestTable(unittest.TestCase): + + def setUp(self,): + self.table_desc = { + 'name': 'mytest', + 'author': 'josuebrunel', + 'apiKeyURL': 'http://josuebrunel.org/api', + 'documentationURL': 'http://josuebrunel.org/doc.html', + 'sampleQuery': 'SELECT * FROM mytable', + } + + self.table = Table(**self.table_desc) + + self.binder_desc = { + 'name': 'select', + 'itemPath': 'products.product', + 'produces': 'xml' + } + + self.binder = Binder(**self.binder_desc) + self.binder_insert = Binder('insert','products.product','json') + + self.key_desc = { + 'id': 'artist', + 'type': 'xs:string', + 'paramType': 'path' + } + + self.key = BinderKey(**self.key_desc) + self.key2 = BinderKey(id='song', type='xs:string', paramType='path', required='true') + + start= {'id': 'ItemPage', 'default': '1'} + pageSize= {'id':'Count' ,'max':'25'} + total= {'default': '10'} + self.paging = BinderPage('page', start, pageSize, total) + + def xml_pretty_print(self, data): + """Pretty logging.debug xml data + """ + raw_string = xtree.tostring(data, 'utf-8') + parsed_string = minidom.parseString(raw_string) + return parsed_string.toprettyxml(indent='\t') + + def test_add_binder(self,): + self.assertEqual(self.table.addBinder(self.binder),True) + logging.debug(self.xml_pretty_print(self.table.etree)) + + def test_remove_binder(self,): + self.binder.addInput(self.key) + self.binder_insert.addInput(self.key) + self.binder.addFunction('', from_file='tests_data/jscode.js') + self.binder_insert.addFunction("console.log('hello this is an insert function'); ") + self.table.addBinder(self.binder) + self.table.addBinder(self.binder_insert) + self.table.save(name='before', path='tests_data') + self.assertEqual(os.path.isfile('tests_data/before.xml'),True) + self.table.removeBinder('select') + self.table.save(name='after', path='tests_data') + self.assertEqual(os.path.isfile('tests_data/after.xml'),True) + + def test_add_input_to_binder(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEqual(self.binder.addInput(self.key),True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_remove_input(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEqual(self.binder.addInput(self.key),True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEqual(self.binder.addInput(self.key2),True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEquals(self.binder.removeInput(key_id='artist'),True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_add_function_from_file(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_remove_function(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEqual(self.binder.addFunction('', from_file='tests_data/jscode.js'),True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEqual(self.binder.removeFunction(), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_add_paging(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEquals(self.binder.addPaging(self.paging), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_create_table_with_paging(self,): + start= {'id': 'ItemPage', 'default': '1'} + pageSize= {'id':'Count' ,'max':'25'} + total= {'default': '10'} + paging = BinderPage('page', start, pageSize, total) + logging.debug(self.xml_pretty_print(paging.etree)) + self.binder_desc['paging']=paging + logging.debug(self.binder_desc) + binder = Binder(**self.binder_desc) + self.assertNotEqual(binder.paging,None) + logging.debug(self.xml_pretty_print(binder.etree)) + + def test_remove_paging(self,): + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEquals(self.binder.addPaging(self.paging), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEqual(self.binder.removePaging(), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_create_binder_with_urls(self,): + url = 'http://josuebrunel.org/service/v1' + url2 = 'http://josuebrunel.org/service/v1/?name=lol' + self.binder_desc['urls'] = [url, url2] + binder = Binder(**self.binder_desc) + logging.debug(self.xml_pretty_print(binder.etree)) + #self.assertEquals(self.binder.addUrl(url), True) + #logging.debug(self.xml_pretty_print(self.binder.etree)) + + + def test_add_url(self,): + url = 'http://josuebrunel.org/service.js' + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEquals(self.binder.addUrl(url), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_remove_url(self,): + url = 'http://josuebrunel.org/service.js' + url2 = 'http://google.com' + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEquals(self.binder.addUrl(url), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEquals(self.binder.addUrl(url2), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEquals(self.binder.removeUrl(url), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) + + def test_save_file(self,): + self.table.save() + self.assertEquals(os.path.isfile('mytest.xml'),True) + + def test_save_with_another_name(self): + name = "tests_data/toto" + self.table.save(name) + self.assertEquals(os.path.isfile(name+'.xml'),True) + + def test_save_to_different_location(self,): + fname = "titi" + path = 'tests_data' + name = os.path.join(path,fname) + self.table.save(name=fname, path=path) + self.assertEquals(os.path.isfile(name+'.xml'),True) + + def test_create_table(self,): + self.binder.addInput(self.key) + self.binder.addFunction('', from_file='tests_data/jscode.js') + logging.debug(self.xml_pretty_print(self.table.etree)) + self.table.addBinder(self.binder) + self.table.save(name='mytable', path='tests_data') + self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + logging.debug(self.xml_pretty_print(self.table.etree)) + + def test_create_table_and_add_two_binders(self,): + self.binder.addInput(self.key) + self.binder_insert.addInput(self.key) + self.binder.addFunction('', from_file='tests_data/jscode.js') + self.binder_insert.addFunction("console.log('hello this is an insert function'); ") + logging.debug(self.xml_pretty_print(self.table.etree)) + self.table.addBinder(self.binder) + logging.debug(self.xml_pretty_print(self.table.etree)) + self.table.addBinder(self.binder_insert) + self.table.save(name='mytable', path='tests_data') + self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + logging.debug(self.xml_pretty_print(self.table.etree)) + + def test_create_table_with_binder(self,): + self.binder.addInput(self.key) + self.binder.addFunction('', from_file='tests_data/jscode.js') + self.table_desc['bindings'] = [self.binder] + table = Table(**self.table_desc) + logging.debug(self.xml_pretty_print(table.etree)) + table.save(name='mytable', path='tests_data') + self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + logging.debug(self.xml_pretty_print(table.etree)) + + def test_create_table_with_two_binders(self,): + self.binder.addInput(self.key) + self.binder.addFunction('', from_file='tests_data/jscode.js') + self.table_desc['bindings'] = [self.binder, self.binder_insert] + table = Table(**self.table_desc) + logging.debug(self.xml_pretty_print(table.etree)) + table.save(name='mytable', path='tests_data') + self.assertEqual(os.path.isfile('tests_data/mytable.xml'),True) + logging.debug(self.xml_pretty_print(table.etree)) + + def test_add_function_table(self): + logging.debug(self.xml_pretty_print(self.table.etree)) + self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) + logging.debug(self.xml_pretty_print(self.table.etree)) + + def test_remove_function_table(self,): + logging.debug(self.xml_pretty_print(self.table.etree)) + self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) + logging.debug(self.xml_pretty_print(self.table.etree)) + self.assertEquals(self.table.removeFunction(),True) + logging.debug(self.xml_pretty_print(self.table.etree)) + + def tearUp(self): + os.path.unlink('tests_data/mytest.xml') + os.path.unlink('tests_data/toto.xml') + +if '__main__' == __name__: + unittest.main() -from myql import MYQL -from myql.errors import NoTableSelectedError -#from myql.contrib import YahooOAuth - -from test_config import consumer_key, consumer_secret - -class LokingYqlTest(unittest.TestCase): - - def setUp(self,): - self.yql = MYQL() - #self.yoauth = YahooOAuth(consumer_key, consumer_secret) - - def tearUp(self,): - pass - - def test_use(self, url= 'http://myserver.com/mytables.xml'): - '''Tests use method - ''' - self.yql.use(url) - self.assertEquals(self.yql.url, url) - - # def test_select_where(self, table='geo.states', items=['']) - -#def test_config(self,): - # '''Tests test config file - # ''' - # conf = self.yql.loadConfig('tests.test_config') - - # self.assertEquals('josue', conf.consumer_key) - # self.assertEquals('brunel', conf.consumer_secret) - - #def test_desc_with_no_table(self,): - #self.assertRaises(NoTableSelectedError, self.yql.desc()) - - def test_desc_with_table(self,): - pass - -# def test_oauth(self,): -# #Step 1: Getting the request token -# self.yoauth.get_request_token() -# #Step 2: Getting the user authorization -# self.yoauth.get_user_authorization() -# #Step 3: Getting access token -# self.yoauth.get_access_token() - - -if __name__ == "__main__": - unittest.main() From 97e022553192cb9508983abbd93ba10503666f34 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Apr 2015 15:28:32 +0200 Subject: [PATCH 140/582] Adding js code --- tests_data/jscode.js | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 tests_data/jscode.js diff --git a/tests_data/jscode.js b/tests_data/jscode.js new file mode 100755 index 0000000..f3b503b --- /dev/null +++ b/tests_data/jscode.js @@ -0,0 +1,5 @@ + // Include the flickr signing library + y.include("http:blog.pipes.yahoo.net/wp-content/uploads/flickr.js"); + // GET the flickr result using a signed url + var fs = new flickrSigner(api_key,secret); + response.object = y.rest(fs.createUrl({method:method, format:""})).get().response(); From cdc5498c88a8a6514f01d7f3a4ff3061abaa82af Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Apr 2015 15:36:26 +0200 Subject: [PATCH 141/582] travis is back on rails --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..15176ce --- /dev/null +++ b/.travis.yml @@ -0,0 +1,5 @@ +language: python +python: + - "2.7" +install: "pip install -r requirements.txt" +script: ./run_tests.sh From 181a7cf2842af589bf335dbb0e403f57cdf72513 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Apr 2015 15:45:03 +0200 Subject: [PATCH 142/582] Adding travis badge to projects --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b715812..aabceac 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ MYQL ========= +[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) + MYQL is a Python wrapper of the Yahoo Query Language. Yahoo! Query Langauge Documentation and Support @@ -12,9 +14,10 @@ Yahoo! Query Langauge Documentation and Support * Yahoo! Social APIs - http://developer.yahoo.com/social/ * Yahoo! QUery Language Console https://developer.yahoo.com/yql/console/ -version 1.0 +version 1.2 =========== +* OpenTable classes * Access to resources requiring authentication version 0.5.6 From b781e2fadca5cdeb8c9ea232497d4c546ab09a44 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 9 Apr 2015 10:01:40 +0200 Subject: [PATCH 143/582] Fix #44 : Binding a Table with a binder isn't that ugly anymore --- myql/contrib/table/table.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index e4c3846..6b34b96 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -2,7 +2,7 @@ from xml.dom import minidom from xml.etree import cElementTree as xtree -from binder import BinderMeta +from binder import BinderMeta, Binder class Table(object): """Class representating a YQL Table @@ -169,7 +169,8 @@ class TableMeta(type): def __new__(cls, name, bases, dct): if name != 'TableModel': table_attr = {key: value for (key, value) in dct.items() if key in cls.TABLE_KEYS } - table_attr['bindings'] = [ value.binder for value in dct.values() if hasattr(value, 'binder') and isinstance(value, BinderMeta) ] + #table_attr['bindings'] = [ value.binder for value in dct.values() if hasattr(value, 'binder') and isinstance(value, BinderMeta) ] + table_attr['bindings'] = [ value for value in dct.values() if isinstance(value, Binder) ] dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} dct['table'] = Table(**table_attr) @@ -181,3 +182,5 @@ def toxml(cls,): class TableModel(Table): __metaclass__ = TableMeta +def BinderFrom(cls_binder_meta): + return cls_binder_meta.binder From 27fe7f4880167e1754006a8b1f36f4ae2b837268 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 9 Apr 2015 10:18:50 +0200 Subject: [PATCH 144/582] fix #45 : several sampleQuery supported now --- myql/contrib/table/table.py | 7 ++++--- tests/tests.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index 6b34b96..e58218f 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -10,7 +10,7 @@ class Table(object): _TAB_ATTR = {'xmlns':'http://query.yahooapis.com/v1/schema/table.xsd', 'securityLevel':'any', 'https':'false'} - def __init__(self, name, author, apiKeyURL, documentationURL, sampleQuery, description=None, table_attr=None, bindings=[]): + def __init__(self, name, author, apiKeyURL, documentationURL, sampleQuery=[], description=None, table_attr=None, bindings=[]): """Initialize the class """ self.name = name @@ -81,8 +81,9 @@ def _init_table_elementTree(self, xml=True, db_table=True): ## ## - t_sampleQuery = xtree.SubElement(t_meta, 'sampleQuery') - t_sampleQuery.text = self.sampleQuery + for sample_query in self.sampleQuery: + t_sampleQuery = xtree.SubElement(t_meta, 'sampleQuery') + t_sampleQuery.text = sample_query ## ## diff --git a/tests/tests.py b/tests/tests.py index 99a9aa2..8678a8a 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -20,7 +20,7 @@ def setUp(self,): 'author': 'josuebrunel', 'apiKeyURL': 'http://josuebrunel.org/api', 'documentationURL': 'http://josuebrunel.org/doc.html', - 'sampleQuery': 'SELECT * FROM mytable', + 'sampleQuery': ['SELECT * FROM mytable', 'SELECT name FROM mytable WHERE id="345"','DELETE FROM mytable WHERE id="345"'], } self.table = Table(**self.table_desc) @@ -103,7 +103,7 @@ def test_add_paging(self,): self.assertEquals(self.binder.addPaging(self.paging), True) logging.debug(self.xml_pretty_print(self.binder.etree)) - def test_create_table_with_paging(self,): + def test_create_binder_with_paging(self,): start= {'id': 'ItemPage', 'default': '1'} pageSize= {'id':'Count' ,'max':'25'} total= {'default': '10'} From c129276b324133a93c23631c46efb287a99ebe9d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 9 Apr 2015 18:15:09 +0200 Subject: [PATCH 145/582] description taken into account --- myql/contrib/table/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index e58218f..f0e5204 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -165,7 +165,7 @@ def removeFunction(self): class TableMeta(type): - TABLE_KEYS = ['name', 'author', 'apiKeyURL', 'documentationURL', 'sampleQuery'] + TABLE_KEYS = ['name', 'author', 'apiKeyURL', 'documentationURL', 'sampleQuery','description'] def __new__(cls, name, bases, dct): if name != 'TableModel': From f7b1d93a5d007ad689abfd5700d1efecb97f88a2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 10 Apr 2015 17:16:49 +0200 Subject: [PATCH 146/582] #49: In progress. Tag order is the main issue here --- myql/contrib/table/table.py | 63 ++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index f0e5204..22c4986 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -59,32 +59,43 @@ def _init_table_elementTree(self, xml=True, db_table=True): # t_meta = xtree.SubElement(t_table, 'meta') - - ## - t_author = xtree.SubElement(t_meta, 'author') - t_author.text = self.author - ## - - ## - t_apiKeyURL = xtree.SubElement(t_meta, 'apiKeyURL') - t_apiKeyURL.text = self.apiKey - ## - - ## - t_documentationURL = xtree.SubElement(t_meta, 'documentationURL') - t_documentationURL.text = self.documentationURL - ## - - ## - t_description = xtree.SubElement(t_meta, 'description') - t_description.text = self.description - ## - - ## - for sample_query in self.sampleQuery: - t_sampleQuery = xtree.SubElement(t_meta, 'sampleQuery') - t_sampleQuery.text = sample_query - ## + + for key, value in [(k,v) for k,v in self.__dict__.items() if k != 'table_attr' ]: + #import pdb + #pdb.set_trace() + if isinstance(value, list): + for elt in value: + t_tag = xtree.SubElement(t_meta, key) + t_tag.text = elt + else: + t_tag = xtree.SubElement(t_meta,key) + t_tag.text = value + + # ## + # t_author = xtree.SubElement(t_meta, 'author') + # t_author.text = self.author + # ## + + # ## + # t_apiKeyURL = xtree.SubElement(t_meta, 'apiKeyURL') + # t_apiKeyURL.text = self.apiKey + # ## + + # ## + # t_documentationURL = xtree.SubElement(t_meta, 'documentationURL') + # t_documentationURL.text = self.documentationURL + # ## + + # ## + # t_description = xtree.SubElement(t_meta, 'description') + # t_description.text = self.description + # ## + + # ## + # for sample_query in self.sampleQuery: + # t_sampleQuery = xtree.SubElement(t_meta, 'sampleQuery') + # t_sampleQuery.text = sample_query + # ## ## t_bindings = xtree.SubElement(t_table, 'bindings') From 1858e4d4bfe64e7f0f0475b0d92b773e87063914 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 11 Apr 2015 04:27:28 +0200 Subject: [PATCH 147/582] fix #49: A lil way more pythonic --- myql/contrib/table/table.py | 41 +++++++------------------------------ 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index 22c4986..6f8d4f1 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -59,44 +59,17 @@ def _init_table_elementTree(self, xml=True, db_table=True): # t_meta = xtree.SubElement(t_table, 'meta') - - for key, value in [(k,v) for k,v in self.__dict__.items() if k != 'table_attr' ]: - #import pdb - #pdb.set_trace() - if isinstance(value, list): + + # Loop on a sorted key,value of class attributes while ignoring table_attr and name + for key, value in [(k,v) for k,v in sorted(self.__dict__.items(), key=lambda x: x[0]) if k not in ('table_attr','name') ]: + if isinstance(value, list): # Works for element like sampleQuery for elt in value: - t_tag = xtree.SubElement(t_meta, key) - t_tag.text = elt + t_tag = xtree.SubElement(t_meta, key) # setting attribute name as a tag name + t_tag.text = elt # Setting attribute value as text else: t_tag = xtree.SubElement(t_meta,key) t_tag.text = value - - # ## - # t_author = xtree.SubElement(t_meta, 'author') - # t_author.text = self.author - # ## - - # ## - # t_apiKeyURL = xtree.SubElement(t_meta, 'apiKeyURL') - # t_apiKeyURL.text = self.apiKey - # ## - - # ## - # t_documentationURL = xtree.SubElement(t_meta, 'documentationURL') - # t_documentationURL.text = self.documentationURL - # ## - - # ## - # t_description = xtree.SubElement(t_meta, 'description') - # t_description.text = self.description - # ## - - # ## - # for sample_query in self.sampleQuery: - # t_sampleQuery = xtree.SubElement(t_meta, 'sampleQuery') - # t_sampleQuery.text = sample_query - # ## - + ## t_bindings = xtree.SubElement(t_table, 'bindings') ## From 3032d334b79a4f24f860d40f65d5ab683db8cecb Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 11 Apr 2015 11:09:21 +0200 Subject: [PATCH 148/582] fix #48: Table and Binder inherits from Base class --- myql/contrib/table/binder.py | 36 ++---------------------------------- myql/contrib/table/table.py | 36 +++--------------------------------- 2 files changed, 5 insertions(+), 67 deletions(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index b4ccc2b..5bd0b86 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -1,6 +1,7 @@ +from base import Base from xml.etree import cElementTree as xtree -class Binder(object): +class Binder(Base): """Class describing binders : select, insert, update, delete name : select, insert, update, delete itemPath : dotted path i.e : products.product @@ -115,39 +116,6 @@ def removeInput(self, key_id): print(e) return False - def addFunction(self, function_code, from_file=''): - """Adds function section to the binder - """ - root = self.etree - - t_execute = root.find('execute') - - if not t_execute: - t_execute = xtree.SubElement(root, 'execute') - - if from_file : - with open(from_file) as f: - function_code = f.read() - - #t_execute.text = function_code - t_execute.text = "\n ![CDATA[ {0} ]] \n".format(function_code) - - return True - - def removeFunction(self,): - """Removes function of the binder - """ - root = self.etree - t_execute = root.find('execute') - - try: - root.remove(t_execute) - return True - except Exception, e: - print(e) - - return False - def addPaging(self, paging): """Adds paging to binder """ diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index 6f8d4f1..9a29a2f 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -2,9 +2,10 @@ from xml.dom import minidom from xml.etree import cElementTree as xtree +from base import Base from binder import BinderMeta, Binder -class Table(object): +class Table(Base): """Class representating a YQL Table """ @@ -60,7 +61,7 @@ def _init_table_elementTree(self, xml=True, db_table=True): # t_meta = xtree.SubElement(t_table, 'meta') - # Loop on a sorted key,value of class attributes while ignoring table_attr and name + # Loop over a sorted key,value of class attributes while ignoring table_attr and name for key, value in [(k,v) for k,v in sorted(self.__dict__.items(), key=lambda x: x[0]) if k not in ('table_attr','name') ]: if isinstance(value, list): # Works for element like sampleQuery for elt in value: @@ -115,37 +116,6 @@ def removeBinder(self, name): return False - def addFunction(self, func_code, from_file=None): - """Adds function to a YQL Table - """ - - if from_file: - with open(from_file) as f: - func_code = f.read() - - root = self.etree - - t_execute = xtree.SubElement(root, 'execute') - t_execute.text = "\n [!CDATA[ {0} ]]\n".format(func_code) - - return True - - def removeFunction(self): - """Removes from a table - """ - - root = self.etree - - t_execute = root.find('execute') - - try : - root.remove(t_execute) - return True - except Execption,e: - print(e) - - return False - class TableMeta(type): From fc7ddfc945193ec04ba7217656debd6b829d907a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 11 Apr 2015 11:15:31 +0200 Subject: [PATCH 149/582] #48: fix import of base module --- myql/contrib/table/base.py | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 myql/contrib/table/base.py diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py new file mode 100644 index 0000000..5592db8 --- /dev/null +++ b/myql/contrib/table/base.py @@ -0,0 +1,39 @@ +from xml.etree import cElementTree as ctree + +class Base(object): + + def addElement(self, elt, tagName=None): + pass + + def removeElement(self, elt, tagName=None): + pass + + def addFunction(self, func_code, from_file=''): + """Add function + """ + if from_file: + with open(from_file) as f: + func_code = f.read() + + root = self.etree + t_execute = root.find('execute') + + if not t_execute : + t_execute = ctree.SubElement(root, 'execute') + + t_execute.text = "\n ![CDATA] {0} ]]\n".format(func_code) + + return True + + def removeFunction(self): + """Remove function tag + """ + root = self.etree + t_execute = root.find('execute') + try: + root.remove(t_execute) + return True + except Exception,e: + print(e) + + return False From 52b508d7baad32e689bdd2406aec6558f0196481 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 11 Apr 2015 11:47:12 +0200 Subject: [PATCH 150/582] handling like attribute for BinderKey --- myql/contrib/table/binder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 5bd0b86..536a5db 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -157,6 +157,8 @@ def __init__(self, id, type, paramType, required='false', like=''): self.type = type self.paramType = paramType self.required = required + if like: + self.like = like self.etree = self._buildElementTree() From 6b93187ab1146c4cee55be7aee20edaab329d64f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 11 Apr 2015 12:20:37 +0200 Subject: [PATCH 151/582] Generic add/removeElement added --- myql/contrib/table/base.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index 5592db8..25c048e 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -3,10 +3,25 @@ class Base(object): def addElement(self, elt, tagName=None): - pass + root = self.etree + t_elt = root.find(tagName) + if not t_elt: + t_elt = ctree.SubElement(root, tagName) + + root.append(t_elt) + return True def removeElement(self, elt, tagName=None): - pass + root = self.etree + t_elt = root.find(tagName) + + try: + root.remove(t_elt) + return True + except Exception,e: + print(e) + + return False def addFunction(self, func_code, from_file=''): """Add function From a96b19bec81ed3b745db5a16ed2408c45f21464b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 11 Apr 2015 13:20:08 +0200 Subject: [PATCH 152/582] #50 OK for binders --- myql/contrib/table/binder.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 536a5db..07a766e 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -215,8 +215,11 @@ def __new__(cls, name, bases, dct): if paging : binder_attr['paging'] = paging[0] + binder = Binder(**binder_attr) + if dct.get('function',None): + binder.addFunction(func_code='', from_file=dct['function']) dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} - dct['binder'] = Binder(**binder_attr) + dct['binder'] = binder # Add KeyException Management return super(BinderMeta,cls).__new__(cls, name, (Binder,), dct) From 4e28b93b08468fc8902b92cd8eb22bd815719cc1 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 11 Apr 2015 13:46:14 +0200 Subject: [PATCH 153/582] fix #50 : Support of function added --- myql/contrib/table/base.py | 2 +- myql/contrib/table/table.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index 25c048e..bde4f2a 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -36,7 +36,7 @@ def addFunction(self, func_code, from_file=''): if not t_execute : t_execute = ctree.SubElement(root, 'execute') - t_execute.text = "\n ![CDATA] {0} ]]\n".format(func_code) + t_execute.text = "\n\t![CDATA]{0:>5}]]\n\t".format(func_code.ljust(4)) return True diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index 9a29a2f..d67a026 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -124,10 +124,12 @@ class TableMeta(type): def __new__(cls, name, bases, dct): if name != 'TableModel': table_attr = {key: value for (key, value) in dct.items() if key in cls.TABLE_KEYS } - #table_attr['bindings'] = [ value.binder for value in dct.values() if hasattr(value, 'binder') and isinstance(value, BinderMeta) ] table_attr['bindings'] = [ value for value in dct.values() if isinstance(value, Binder) ] + table = Table(**table_attr) + if dct.get('function', None): + table.addFunction(func_code='', from_file=dct['function']) dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} - dct['table'] = Table(**table_attr) + dct['table'] = table return super(TableMeta, cls).__new__(cls, name, (Table,), dct) From 309d1facd7abe80c0e6e3a2667c730f0a1c57f97 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 12 Apr 2015 23:02:54 +0200 Subject: [PATCH 154/582] MANIFEST file added --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f9bd145 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include requirements.txt From 85093af8e188a5c80e9882c8c2456774311b4e83 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 10:30:09 +0200 Subject: [PATCH 155/582] #51 : InputBase class created --- myql/contrib/table/base.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index bde4f2a..952e0ed 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -52,3 +52,16 @@ def removeFunction(self): print(e) return False + +class InputBase(object): + + def __init__(self, id, type, paramType, like='', required='false', default='', private='false', const='false', batchable='false', maxbatchItem=5,): + self.id = id + self.as = like + self.required = required + self.default = default + self.private = private + self.const = const + self.batchable = batchable + self.maxbatchItem = maxbatchItem + From 9d4664176a63debddbea4e9b8e0f9b987c6ef8c5 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 11:56:28 +0200 Subject: [PATCH 156/582] adding base classes to table package --- myql/contrib/table/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/myql/contrib/table/__init__.py b/myql/contrib/table/__init__.py index ca3c7c4..93f0c64 100755 --- a/myql/contrib/table/__init__.py +++ b/myql/contrib/table/__init__.py @@ -1,2 +1,3 @@ from table import * from binder import * +from base import * From 4be0b57e3023fac90eeccfdaa55bc8a12b39068a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 12:11:49 +0200 Subject: [PATCH 157/582] #51 : BaseInput Manuel test added --- tests/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 8678a8a..8ca697e 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -3,8 +3,9 @@ import unittest from xml.dom import minidom from xml.etree import cElementTree as xtree -from myql.contrib.table import Binder, BinderKey, BinderPage from myql.contrib.table import Table +from myql.contrib.table import Base, BaseInput +from myql.contrib.table import Binder, BinderKey, BinderPage import readline, rlcompleter readline.parse_and_bind('tab: complete') @@ -219,6 +220,10 @@ def test_remove_function_table(self,): self.assertEquals(self.table.removeFunction(),True) logging.debug(self.xml_pretty_print(self.table.etree)) + def test_baseinput_to_xml(self,): + i = BaseInput('key','name','xs:string', 'path', required=True, default='josh', private=True, maxBatchItems=10) + logging.debug(self.xml_pretty_print(i.etree)) + def tearUp(self): os.path.unlink('tests_data/mytest.xml') os.path.unlink('tests_data/toto.xml') From 07efcd43520a48512de8094dd0bddaf34df68183 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 12:18:06 +0200 Subject: [PATCH 158/582] #51 BaseInput Class created and tested --- myql/contrib/table/base.py | 41 ++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index 952e0ed..c1fd2fd 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -53,15 +53,48 @@ def removeFunction(self): return False -class InputBase(object): +class BaseInput(object): + """This class represents an Input Element under Binding element. + Input Element can be : , or + Full Documentation https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-key + """ - def __init__(self, id, type, paramType, like='', required='false', default='', private='false', const='false', batchable='false', maxbatchItem=5,): + def __init__(self, input_type, id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=5,): + """ + - input_type : can be , or + - id : The name of the key. This represents what the user needs to provide in the WHERE clause. + - like (as): The alias of the key used in YQL statements. + - type : The type of data coming back from the Web service. + - required : A boolean that answers the question + - paramType : Determines how this key is represented and passed on to the Web service. + - default : This value is used if one isn't specified by the developer in the SELECT. + - private : Hide this key's value to the user (in both "desc" and "diagnostics"). + - const : A boolean that indicates whether the default attribute must be present and cannot be changed by the end user. + - batchable : A boolean which answers the question: Does this select and URL support multiple key fetches/requests in a single request (batched fetching)? + - maxBatchItems : How many requests should be combined in a single batch call. + """ + self.name = input_type self.id = id - self.as = like + self.like = like # as is a python , that's why in argument we use + self.type = type self.required = required self.default = default self.private = private self.const = const self.batchable = batchable - self.maxbatchItem = maxbatchItem + self.maxBatchItems = maxBatchItems + + self.etree = self._buildElementTree() + + def _buildElementTree(self,): + """Turn object into an ElementTree + """ + t_elt = ctree.Element(self.name) + + for k,v in [ (key,value) for key,value in self.__dict__.items() if key != 'name']: # Excluding name from list of items + if v and v != 'false' : + t_elt.set(k if k != 'like' else 'as', str(v).lower()) + + self._etree = t_elt + return t_elt From 8d62321c4503bdab11cb7a9ca6e3333d04265e19 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:02:36 +0200 Subject: [PATCH 159/582] 51 : BinderKey now subclassing BaseInput --- myql/contrib/table/binder.py | 54 ++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 07a766e..ed41479 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -1,4 +1,4 @@ -from base import Base +from base import Base, BaseInput from xml.etree import cElementTree as xtree class Binder(Base): @@ -146,36 +146,36 @@ def removePaging(self,): return False -class BinderKey(object): +#class BinderKey(object): +class BinderKey(BaseInput): """Class representing a key which is part of inputs """ - def __init__(self, id, type, paramType, required='false', like=''): - """Initializes the class - """ - self.id = id - self.type = type - self.paramType = paramType - self.required = required - if like: - self.like = like - - self.etree = self._buildElementTree() - - def _buildElementTree(self,): - """Turns object into ElementTre - """ - t_key = xtree.Element('key') - for item in self.__dict__.items(): - t_key.set(*item) - - return t_key + #def __init__(self, id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=5,): + def __init__(self, *args, **kwargs): + + super(BinderKey, self).__init__('key', *args, **kwargs) +# def __init__(self, id, type, paramType, required='false', like=''): +# """Initializes the class +# """ +# self.id = id +# self.type = type +# self.paramType = paramType +# self.required = required +# if like: +# self.like = like +# +# self.etree = self._buildElementTree() +# +# def _buildElementTree(self,): +# """Turns object into ElementTre +# """ +# t_key = xtree.Element('key') +# for item in self.__dict__.items(): +# t_key.set(*item) +# +# return t_key - def to_elementTree(self,): - """Returns object as ElementTree - """ - return self.etree - class BinderPage(object): def __init__(self, model, start, pageSize, total): From f68586919813bf1e94cc5ab05785df0aeb06f36c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:07:21 +0200 Subject: [PATCH 160/582] #51: default maxBatchItems changed to 1 --- myql/contrib/table/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index c1fd2fd..bca2931 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -59,7 +59,7 @@ class BaseInput(object): Full Documentation https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-key """ - def __init__(self, input_type, id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=5,): + def __init__(self, input_type, id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=0): """ - input_type : can be , or - id : The name of the key. This represents what the user needs to provide in the WHERE clause. From 1edb8438d1b1eca2fd79a8dc9f08b5d9abfa24dd Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:07:36 +0200 Subject: [PATCH 161/582] #51: default maxBatchItems changed to 1 --- myql/contrib/table/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index bca2931..91dc2d4 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -59,7 +59,7 @@ class BaseInput(object): Full Documentation https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-key """ - def __init__(self, input_type, id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=0): + def __init__(self, input_type, id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=1): """ - input_type : can be , or - id : The name of the key. This represents what the user needs to provide in the WHERE clause. From 700a796fa4bd9a67bd6a11fdf837c88e92f5dc1f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:15:51 +0200 Subject: [PATCH 162/582] fix #53 : InputValue and InputMap classes created OK --- myql/contrib/table/binder.py | 47 +++++++++++++++--------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index ed41479..5df48de 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -7,7 +7,7 @@ class Binder(Base): itemPath : dotted path i.e : products.product produces : json or xml urls : list of urls related to the api - inputs : list of BinderKey object + inputs : list of InputKey object """ def __init__(self, name, itemPath, produces, pollingFrequencySeconds=30, urls=[], inputs=[], paging=None): @@ -146,36 +146,27 @@ def removePaging(self,): return False -#class BinderKey(object): -class BinderKey(BaseInput): - """Class representing a key which is part of inputs +class InputKey(BaseInput): + """Class representing a key of an Input """ - #def __init__(self, id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=5,): def __init__(self, *args, **kwargs): - - super(BinderKey, self).__init__('key', *args, **kwargs) -# def __init__(self, id, type, paramType, required='false', like=''): -# """Initializes the class -# """ -# self.id = id -# self.type = type -# self.paramType = paramType -# self.required = required -# if like: -# self.like = like -# -# self.etree = self._buildElementTree() -# -# def _buildElementTree(self,): -# """Turns object into ElementTre -# """ -# t_key = xtree.Element('key') -# for item in self.__dict__.items(): -# t_key.set(*item) -# -# return t_key + super(InputKey, self).__init__('key', *args, **kwargs) +class InputValue(BaseInput): + """Class representing value under an Input + """ + def __init__(self, *args, **kwargs): + super(InputKey, self).__init__('value', *args, **kwargs) + + +class InputMap(BaseInput): + """Class representing map under an Input + """ + def __init__(self, *args, **kwargs): + super(InputKey, self).__init__('map', *args, **kwargs) + + class BinderPage(object): def __init__(self, model, start, pageSize, total): @@ -210,7 +201,7 @@ def __new__(cls, name, bases, dct): if name != 'BinderModel': binder_attr = {key: value for (key, value) in dct.items() if key in cls.BINDER_KEY} - binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BinderKey)] + binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, InputKey)] paging = [ value for value in dct.values() if isinstance(value, BinderPage)] if paging : binder_attr['paging'] = paging[0] From c2949a01676691dd4aa9a84bb0aa1b45db538f40 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:18:04 +0200 Subject: [PATCH 163/582] #51 : fix indent error --- myql/contrib/table/binder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 5df48de..2654d93 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -156,7 +156,7 @@ def __init__(self, *args, **kwargs): class InputValue(BaseInput): """Class representing value under an Input """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): super(InputKey, self).__init__('value', *args, **kwargs) From ae2e6a2645bc1b24c015bc4a03b506d49ca5b82a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:18:33 +0200 Subject: [PATCH 164/582] #51 : fix indent error --- myql/contrib/table/binder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 2654d93..9e952bc 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -163,7 +163,7 @@ def __init__(self, *args, **kwargs): class InputMap(BaseInput): """Class representing map under an Input """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): super(InputKey, self).__init__('map', *args, **kwargs) From b7e02ce4db916a2383d72507e490622136a4ec50 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:19:21 +0200 Subject: [PATCH 165/582] fix #51: tests updated and OK --- tests/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 8ca697e..d49a8be 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -5,7 +5,7 @@ from xml.etree import cElementTree as xtree from myql.contrib.table import Table from myql.contrib.table import Base, BaseInput -from myql.contrib.table import Binder, BinderKey, BinderPage +from myql.contrib.table import Binder, InputKey, BinderPage import readline, rlcompleter readline.parse_and_bind('tab: complete') @@ -41,8 +41,8 @@ def setUp(self,): 'paramType': 'path' } - self.key = BinderKey(**self.key_desc) - self.key2 = BinderKey(id='song', type='xs:string', paramType='path', required='true') + self.key = InputKey(**self.key_desc) + self.key2 = InputKey(id='song', type='xs:string', paramType='path', required='true') start= {'id': 'ItemPage', 'default': '1'} pageSize= {'id':'Count' ,'max':'25'} From c177a6e430ba9d670a9a0369ca2c883908bcaf34 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:40:19 +0200 Subject: [PATCH 166/582] BinderMeta updated --- myql/contrib/table/binder.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 9e952bc..b55f9d7 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -157,14 +157,14 @@ class InputValue(BaseInput): """Class representing value under an Input """ def __init__(self, *args, **kwargs): - super(InputKey, self).__init__('value', *args, **kwargs) + super(InputValue, self).__init__('value', *args, **kwargs) class InputMap(BaseInput): """Class representing map under an Input """ def __init__(self, *args, **kwargs): - super(InputKey, self).__init__('map', *args, **kwargs) + super(InputMap, self).__init__('map', *args, **kwargs) class BinderPage(object): @@ -195,13 +195,15 @@ def __buildElementTree(self,): class BinderMeta(type): - BINDER_KEY = ['name', 'itemPath', 'produces', 'pollingFrequencySeconds', 'urls', 'keys', 'pages'] + INPUT_KEY = ['name', 'itemPath', 'produces', 'pollingFrequencySeconds', 'urls', 'keys', 'pages'] def __new__(cls, name, bases, dct): if name != 'BinderModel': - binder_attr = {key: value for (key, value) in dct.items() if key in cls.BINDER_KEY} - binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, InputKey)] + binder_attr = {key: value for (key, value) in dct.items() if key in cls.INPUT_KEY} + import pdb + pdb.set_trace() + binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BaseInput)] paging = [ value for value in dct.values() if isinstance(value, BinderPage)] if paging : binder_attr['paging'] = paging[0] From cdca60d2a2230c756a94600f57dbf9fe70c84a29 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 13:41:22 +0200 Subject: [PATCH 167/582] removing pdb --- myql/contrib/table/binder.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index b55f9d7..323d5b4 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -201,8 +201,6 @@ def __new__(cls, name, bases, dct): if name != 'BinderModel': binder_attr = {key: value for (key, value) in dct.items() if key in cls.INPUT_KEY} - import pdb - pdb.set_trace() binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BaseInput)] paging = [ value for value in dct.values() if isinstance(value, BinderPage)] if paging : From 6ddeef35b5c2083ffd754577c03e5cabf9e08513 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 14:42:38 +0200 Subject: [PATCH 168/582] InputValue test added --- tests/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index d49a8be..4fa0667 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -5,7 +5,7 @@ from xml.etree import cElementTree as xtree from myql.contrib.table import Table from myql.contrib.table import Base, BaseInput -from myql.contrib.table import Binder, InputKey, BinderPage +from myql.contrib.table import Binder, InputKey, InputValue, BinderPage import readline, rlcompleter readline.parse_and_bind('tab: complete') @@ -223,6 +223,10 @@ def test_remove_function_table(self,): def test_baseinput_to_xml(self,): i = BaseInput('key','name','xs:string', 'path', required=True, default='josh', private=True, maxBatchItems=10) logging.debug(self.xml_pretty_print(i.etree)) + + def test_inputvalue(self,): + v = InputValue('content', 'xs:string', 'variable', required=True) + logging.debug(self.xml_pretty_print(v.etree)) def tearUp(self): os.path.unlink('tests_data/mytest.xml') From 06c310956554aee8d43a1959d351d354f9f608f9 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 14 Apr 2015 17:49:33 +0200 Subject: [PATCH 169/582] #54: BasePaging class added --- myql/contrib/table/base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index 91dc2d4..77580eb 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -98,3 +98,13 @@ def _buildElementTree(self,): self._etree = t_elt return t_elt +class BasePaging(object): + """Class representing a element under a ,,, and + """ + def __init__(self, name, **kwargs): + """ + """ + self.name = name + vars(self).update(kwargs) + + self.etree = self._buildElementTree() + + if self.urls: + [ self.addUrl(url) for url in self.urls ] + + if self.inputs: + [ self.addInput(elt) for elt in self.inputs ] + + if self.paging: + self.addPaging(self.paging) + + def _buildElementTree(self,): + """Turns object into a Element Tree + """ + t_binder = ctree.Element(self.name) + + for k,v in self.__dict__.items(): + if k not in ('name', 'urls', 'inputs', 'paging') and v : + t_binder.set(k,v) + + self.etree = t_binder + return t_binder + + def addUrl(self, url): + """Add url to binder + """ + + if not url in self.urls: + self.urls.append(url) + + root = self.etree + t_urls = root.find('urls') + + if not t_urls: + t_urls = ctree.SubElement(root, 'urls') + + t_url = ctree.SubElement(t_urls, 'url') + t_url.text = url + + return True + + def removeUrl(self, url): + """Remove passed url from a binder + """ + + root = self.etree + t_urls = root.find('urls') + + if not t_urls: + return False + + for t_url in t_urls.findall('url'): + if t_url.text == url.strip(): + t_urls.remove(t_url) + if url in self.urls: + self.urls.remove(url) + return True + + return False + + def addInput(self, key): + """Add key to input : key, value or map + """ + if not key in self.inputs: + self.inputs.append(key) + + root = self.etree + t_inputs = root.find('inputs') + + if not t_inputs : + t_inputs = ctree.SubElement(root, 'inputs') + + t_inputs.append(key.etree) + + return True + + def removeInput(self, key_id): + """Remove key (key, value, map) from Input + """ + root = self.etree + t_inputs = root.find('inputs') + + if not t_inputs: + return False + + keys = t_inputs.findall('key') + + key = [ key for key in keys if key.get('id') == key_id ] + + try: + t_inputs.remove(key) + return True + except Exception,e: + print(e) + + return False + + def addPaging(self,paging): + """Add paging to Binder + """ + if not self.paging: + self.paging = paging + import pdb + pdb.set_trace() + root = self.etree + + try: + root.append(paging.etree) + return True + except Exception,e: + print(e) + + return False + + def removePaging(self, paging): + """Remove paging from Binder + """ + root = self.etree + t_paging = root.find('paging') + + try: + root.remove(t_paging) + return True + except Exception,e: + print(e) + + return False + + class BaseInput(object): """This class represents an Input Element under Binding element. Input Element can be : , or @@ -102,7 +241,6 @@ class BasePaging(object): """Class representing a element under a , , , + """ def __init__(self, name, itemPath, produces, pollingFrequencySeconds=0, urls=[], inputs=[], paging=None): super(Binder, self).__init__(name, itemPath=itemPath, produces=produces, pollingFrequencySeconds=0, urls=urls, inputs=inputs, paging=paging) +class BinderFunction(BaseBinder): + """Represent a function : + """ + + def __init__(self, func_name, inputs=[]): + """func_name : name of the stored procedure + """ + super(BinderFunction, self).__init__('function', inputs=inputs) + self.func_name = func_name + self.etree.set('name', func_name) + class InputKey(BaseInput): """Class representing a key of an Input """ diff --git a/tests/tests.py b/tests/tests.py index b389767..b1098e2 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -5,7 +5,7 @@ from xml.etree import cElementTree as xtree from myql.contrib.table import Table from myql.contrib.table import Base, BaseInput -from myql.contrib.table import Binder, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset +from myql.contrib.table import Binder, BinderFunction, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset import readline, rlcompleter readline.parse_and_bind('tab: complete') @@ -153,8 +153,8 @@ def test_create_binder_with_urls(self,): self.binder_desc['urls'] = [url, url2] binder = Binder(**self.binder_desc) logging.debug(self.xml_pretty_print(binder.etree)) - #self.assertEquals(self.binder.addUrl(url), True) - #logging.debug(self.xml_pretty_print(self.binder.etree)) + self.assertEquals(self.binder.addUrl(url), True) + logging.debug(self.xml_pretty_print(self.binder.etree)) def test_add_url(self,): @@ -234,7 +234,10 @@ def test_create_table_with_two_binders(self,): def test_add_function_table(self): logging.debug(self.xml_pretty_print(self.table.etree)) - self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) + bf = BinderFunction('concat', [self.key, self.key2]) + bf.addFunction('', from_file='tests_data/jscode.js') + self.table.addBinder(bf) + #self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) logging.debug(self.xml_pretty_print(self.table.etree)) def test_remove_function_table(self,): From ce92119af777ea6e6121886b45ec8f73ccd30b08 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 16 Apr 2015 08:59:50 +0200 Subject: [PATCH 181/582] #56 : tests for BinderFunction created --- tests/tests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index b1098e2..104641c 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -240,6 +240,17 @@ def test_add_function_table(self): #self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) logging.debug(self.xml_pretty_print(self.table.etree)) + def test_create_function_with_func_code(self): + logging.debug(self.xml_pretty_print(self.table.etree)) + bf = BinderFunction('concat', func_code='console.log("hello moron !!!")') + logging.debug(self.xml_pretty_print(bf.etree)) + + def test_create_function_with_func_file(self): + logging.debug(self.xml_pretty_print(self.table.etree)) + bf = BinderFunction('concat', func_file='tests_data/jscode.js') + logging.debug(self.xml_pretty_print(bf.etree)) + + def test_remove_function_table(self,): logging.debug(self.xml_pretty_print(self.table.etree)) self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) From 339d3e914d5a7130b65a59230814294a806a060d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 16 Apr 2015 09:06:15 +0200 Subject: [PATCH 182/582] fix #56 : attributes added to BinderFunction and tested OK --- myql/contrib/table/binder.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index d88b3df..9b2a631 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -13,13 +13,19 @@ class BinderFunction(BaseBinder): """Represent a function : """ - def __init__(self, func_name, inputs=[]): + def __init__(self, func_name, func_code='', func_file=None, inputs=[]): """func_name : name of the stored procedure """ super(BinderFunction, self).__init__('function', inputs=inputs) self.func_name = func_name self.etree.set('name', func_name) + if func_code: + self.addFunction(func_code) + + if func_file: + self.addFunction('', from_file=func_file) + class InputKey(BaseInput): """Class representing a key of an Input """ From 8c7cd38e91d2410ad302b9f62943dc4ab7f6bfd0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 16 Apr 2015 09:08:03 +0200 Subject: [PATCH 183/582] fix add_function_table error --- tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 104641c..21c2232 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -234,7 +234,7 @@ def test_create_table_with_two_binders(self,): def test_add_function_table(self): logging.debug(self.xml_pretty_print(self.table.etree)) - bf = BinderFunction('concat', [self.key, self.key2]) + bf = BinderFunction('concat', inputs=[self.key, self.key2]) bf.addFunction('', from_file='tests_data/jscode.js') self.table.addBinder(bf) #self.assertEquals(self.table.addFunction('', from_file='tests_data/jscode.js'),True) From c94d7bc35d097d09155fe82e649c4667cd092d13 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 16 Apr 2015 09:14:14 +0200 Subject: [PATCH 184/582] fix #57 : TableMeta updated and tested OK --- myql/contrib/table/table.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index d67a026..aa866b8 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -3,7 +3,7 @@ from xml.etree import cElementTree as xtree from base import Base -from binder import BinderMeta, Binder +from binder import BinderMeta, Binder, BinderFunction class Table(Base): """Class representating a YQL Table @@ -124,10 +124,10 @@ class TableMeta(type): def __new__(cls, name, bases, dct): if name != 'TableModel': table_attr = {key: value for (key, value) in dct.items() if key in cls.TABLE_KEYS } - table_attr['bindings'] = [ value for value in dct.values() if isinstance(value, Binder) ] + table_attr['bindings'] = [ value for value in dct.values() if isinstance(value, Binder) or isinstance(value, BinderFunction)] table = Table(**table_attr) - if dct.get('function', None): - table.addFunction(func_code='', from_file=dct['function']) + # if dct.get('function', None): + # table.addFunction(func_code='', from_file=dct['function']) dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} dct['table'] = table From 4227989e2a5556bf275ebb4fb01beba0ddb1c665 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 16 Apr 2015 09:15:05 +0200 Subject: [PATCH 185/582] example.py added for Meta classes tests --- myql/contrib/table/example.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 myql/contrib/table/example.py diff --git a/myql/contrib/table/example.py b/myql/contrib/table/example.py new file mode 100644 index 0000000..eaeb193 --- /dev/null +++ b/myql/contrib/table/example.py @@ -0,0 +1,36 @@ +from binder import BinderModel, InputKey, PagingPage, PagingUrl, InputValue, BinderFunction +from table import TableModel, BinderFrom + +class SelectBinder(BinderModel): + name = 'select' + itemPath = 'products.product' + produces = 'xml' + pollingFrequencySeconds = 30 + urls = ['http://lol.com/services?artist={artis}','http://lol.com/services/song={song}'] + paging = PagingPage({'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) + artist = InputKey(id='artist', type='xs:string', paramType='path') + song = InputKey(id='song', type='xs:string', paramType='path', required='true') + +class InsertBinder(BinderModel): + name = 'insert' + itemPath = 'products.product' + produces = 'xml' + pollingFrequencySeconds = 30 + urls = ['http://lol.com/services?artist={artis}','http://lol.com/services/song={song}'] + paging = PagingUrl(nextpage={'path':'yqlsearch.nextpage'}) + artist = InputKey(id='artist', type='xs:string', paramType='path') + song = InputValue(id='song', type='xs:string', paramType='path', required='true') + + +class TestTable(TableModel): + name = 'Test' + author = 'Josue Kouka' + apiKeyURL = 'http://josuebrunel.org/api' + documentationURL = 'http://josuebrunel.org/doc.html' + description = "Just a test table" + sampleQuery = ['SELECT * FROM mytable','SELECT name FROM mytable WHERE id=4656', "SELECT * FROM mytable WHERE name='Josh'"] + select = BinderFrom(SelectBinder) + insert = BinderFrom(InsertBinder) + func1 = BinderFunction('concat', func_code="console.log('Hello Josh!!!')") + +TestTable.table.save(name='Example') From f7c8ea3238ee480fe542c54e661e0adb23d63b42 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 16 Apr 2015 13:45:05 +0200 Subject: [PATCH 186/582] Doc added --- myql/contrib/table/binder.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 9b2a631..8fcc506 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -14,7 +14,10 @@ class BinderFunction(BaseBinder): """ def __init__(self, func_name, func_code='', func_file=None, inputs=[]): - """func_name : name of the stored procedure + """ + - func_name : name of the stored procedure + - func_code : your js code passed through a str + - func_file : file containing your code """ super(BinderFunction, self).__init__('function', inputs=inputs) self.func_name = func_name @@ -48,13 +51,15 @@ def __init__(self, *args, **kwargs): class PagingPage(BasePaging): - + """Class representing + """ def __init__(self, start, pageSize, total): super(PagingPage, self).__init__('page', start=start, pageSize=pageSize, total=total) class PagingOffset(BasePaging): - + """Class representing + """ def __init__(self, matrix, start, pageSize, total): super(PagingOffset, self).__init__('offset', matrix, start=start, pageSize=pageSize, total=total) self.matrix = str(matrix).lower() @@ -62,7 +67,8 @@ def __init__(self, matrix, start, pageSize, total): class PagingUrl(BasePaging): - + """Class representing + """ def __init__(self, nextpage): super(PagingUrl, self).__init__('url', nextpage=nextpage) @@ -88,9 +94,6 @@ def __new__(cls, name, bases, dct): # Add KeyException Management return super(BinderMeta,cls).__new__(cls, name, (Binder,), dct) - def toxml(cls,): - return xtree.tostring(cls.binder.etree) - class BinderModel(Binder): __metaclass__ = BinderMeta From 78bb085d1ef7b3f0c1478d5233d0cfd5780b5ea2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 17 Apr 2015 06:14:04 +0200 Subject: [PATCH 187/582] INPUT_KEY => INPUT_KEYS --- myql/contrib/table/binder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index 8fcc506..0053e83 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -75,12 +75,12 @@ def __init__(self, nextpage): class BinderMeta(type): - INPUT_KEY = ['name', 'itemPath', 'produces', 'pollingFrequencySeconds', 'urls', 'keys', 'pages'] + INPUT_KEYS = ['name', 'itemPath', 'produces', 'pollingFrequencySeconds', 'urls', 'keys', 'pages'] def __new__(cls, name, bases, dct): if name != 'BinderModel': - binder_attr = {key: value for (key, value) in dct.items() if key in cls.INPUT_KEY} + binder_attr = {key: value for (key, value) in dct.items() if key in cls.INPUT_KEYS} binder_attr['inputs'] = [ value for value in dct.values() if isinstance(value, BaseInput)] paging = [ value for value in dct.values() if isinstance(value, BasePaging)] if paging : From 079c56891268543be4584495e154e93f83302024 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 17 Apr 2015 06:14:22 +0200 Subject: [PATCH 188/582] cleaning the cat house --- myql/contrib/table/table.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index aa866b8..6f97510 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -126,18 +126,15 @@ def __new__(cls, name, bases, dct): table_attr = {key: value for (key, value) in dct.items() if key in cls.TABLE_KEYS } table_attr['bindings'] = [ value for value in dct.values() if isinstance(value, Binder) or isinstance(value, BinderFunction)] table = Table(**table_attr) - # if dct.get('function', None): - # table.addFunction(func_code='', from_file=dct['function']) dct = { key : value for (key, value) in dct.items() if key in ('__module__', '__metaclass__')} dct['table'] = table return super(TableMeta, cls).__new__(cls, name, (Table,), dct) - def toxml(cls,): - return xtree.tostring(cls.table.etree) class TableModel(Table): __metaclass__ = TableMeta + def BinderFrom(cls_binder_meta): return cls_binder_meta.binder From 293cfabb4a55b6b9b65fbed03bd711d9424ffcfc Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 19 Apr 2015 21:46:19 +0200 Subject: [PATCH 189/582] updating requirements --- myql/contrib/yahooauth/auth.py | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 myql/contrib/yahooauth/auth.py diff --git a/myql/contrib/yahooauth/auth.py b/myql/contrib/yahooauth/auth.py new file mode 100644 index 0000000..1895e6e --- /dev/null +++ b/myql/contrib/yahooauth/auth.py @@ -0,0 +1,46 @@ +import pdb +import time +import requests +from requests_oauthlib import OAuth1 +from urlparse import parse_qs +import webbrowser + +class OAuth(object): + + BASE_URL = "http://query.yahooapis.com/v1/yql" + REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token" + ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token" + AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth?oauth_token=" + CALLBACK_URI = 'oob' + + def __init__(self, ck=None, cs=None): + """ + """ + self.ck='dj0yJmk9eFJINERDYWk2M3NkJmQ9WVdrOWEyNW1VRmRGTnpZbWNHbzlNQS0tJnM9Y29uc3VtZXJzZWNyZXQmeD1iNQ--' + self.cs='08802b459ab48eeaca765c119b0af6a4b75789f7' + self.oauth = OAuth1(self.ck, client_secret=self.cs, callback_uri=self.CALLBACK_URI) + + def request_token(self,): + """ + """ + response = requests.post(url=self.REQUEST_TOKEN_URL, auth=self.oauth) + credentias = parse_qs(response.content) + self.request_token = credentias.get('oauth_token')[0] + self.request_token_secret = credentias.get('oauth_token_secret')[0] + print(self.request_token, self.request_token_secret) + return self.request_token, self.request_token_secret + + def get_user_authorization(self,): + """ + """ + webbrowser.open(self.AUTHORIZE_TOKEN_URL+self.request_token) + self.verifier = raw_input("Please input a verifier: ") + print(self.verifier) + + +if '__main__' == __name__: + + auth = OAuth() + auth.request_token() + auth.get_user_authorization() + From 10cf91eb788d2032c6698b5dc491741a728adb95 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 19 Apr 2015 21:48:21 +0200 Subject: [PATCH 190/582] #12 : Request token and verifier OK --- myql/contrib/__init__.py | 2 +- myql/contrib/yahooauth/yahooauth.py | 2 +- requirements.txt | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/myql/contrib/__init__.py b/myql/contrib/__init__.py index b27bbfe..4a3d61d 100755 --- a/myql/contrib/__init__.py +++ b/myql/contrib/__init__.py @@ -1,2 +1,2 @@ -import yahooauth +#import yahooauth import table diff --git a/myql/contrib/yahooauth/yahooauth.py b/myql/contrib/yahooauth/yahooauth.py index 12c6787..fd4e18a 100755 --- a/myql/contrib/yahooauth/yahooauth.py +++ b/myql/contrib/yahooauth/yahooauth.py @@ -1,6 +1,6 @@ import time import webbrowser -from rauth import OAuth1Service +from requests_oauthlib import OAuth1 class YahooOAuth(object): '''OAuth for yahoo api diff --git a/requirements.txt b/requirements.txt index dd5565d..235213f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ certifi==14.05.14 +oauthlib==0.7.2 python-dateutil==2.2 -rauth==0.7.0 requests==2.4.0 -requests-oauthlib==0.4.1 +requests-oauthlib==0.4.2 +six==1.9.0 wsgiref==0.1.2 From f19ffb2785828ffcad1f2034d9362dd6f43003f0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 19 Apr 2015 21:58:32 +0200 Subject: [PATCH 191/582] you know wy --- myql/contrib/yahooauth/auth.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/myql/contrib/yahooauth/auth.py b/myql/contrib/yahooauth/auth.py index 1895e6e..a08e98c 100644 --- a/myql/contrib/yahooauth/auth.py +++ b/myql/contrib/yahooauth/auth.py @@ -4,6 +4,7 @@ from requests_oauthlib import OAuth1 from urlparse import parse_qs import webbrowser +import credentials class OAuth(object): @@ -16,8 +17,8 @@ class OAuth(object): def __init__(self, ck=None, cs=None): """ """ - self.ck='dj0yJmk9eFJINERDYWk2M3NkJmQ9WVdrOWEyNW1VRmRGTnpZbWNHbzlNQS0tJnM9Y29uc3VtZXJzZWNyZXQmeD1iNQ--' - self.cs='08802b459ab48eeaca765c119b0af6a4b75789f7' + self.ck = credentials.ck + self.cs = credentials.cs self.oauth = OAuth1(self.ck, client_secret=self.cs, callback_uri=self.CALLBACK_URI) def request_token(self,): From 2e676dacb36db159b57a043477f027a3b3454a2b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 14:01:23 +0200 Subject: [PATCH 192/582] renaming yahooauth to auth --- myql/contrib/__init__.py | 3 ++- myql/contrib/{yahooauth => auth}/__init__.py | 0 myql/contrib/{yahooauth => auth}/auth.py | 0 myql/contrib/{yahooauth => auth}/readme.md | 0 myql/contrib/{yahooauth => auth}/yahooauth.py | 0 5 files changed, 2 insertions(+), 1 deletion(-) rename myql/contrib/{yahooauth => auth}/__init__.py (100%) rename myql/contrib/{yahooauth => auth}/auth.py (100%) rename myql/contrib/{yahooauth => auth}/readme.md (100%) rename myql/contrib/{yahooauth => auth}/yahooauth.py (100%) diff --git a/myql/contrib/__init__.py b/myql/contrib/__init__.py index 4a3d61d..167809b 100755 --- a/myql/contrib/__init__.py +++ b/myql/contrib/__init__.py @@ -1,2 +1,3 @@ -#import yahooauth +import auth import table + diff --git a/myql/contrib/yahooauth/__init__.py b/myql/contrib/auth/__init__.py similarity index 100% rename from myql/contrib/yahooauth/__init__.py rename to myql/contrib/auth/__init__.py diff --git a/myql/contrib/yahooauth/auth.py b/myql/contrib/auth/auth.py similarity index 100% rename from myql/contrib/yahooauth/auth.py rename to myql/contrib/auth/auth.py diff --git a/myql/contrib/yahooauth/readme.md b/myql/contrib/auth/readme.md similarity index 100% rename from myql/contrib/yahooauth/readme.md rename to myql/contrib/auth/readme.md diff --git a/myql/contrib/yahooauth/yahooauth.py b/myql/contrib/auth/yahooauth.py similarity index 100% rename from myql/contrib/yahooauth/yahooauth.py rename to myql/contrib/auth/yahooauth.py From 2192c59e1c0967f4407f5a997ac3c8ea11378647 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 15:16:35 +0200 Subject: [PATCH 193/582] all OAUTH Steps are OK --- myql/contrib/auth/auth.py | 48 +++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index a08e98c..318eedc 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -6,42 +6,62 @@ import webbrowser import credentials -class OAuth(object): +BASE_URL = "http://query.yahooapis.com/v1/yql" +REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token" +ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token" +AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth?oauth_token=" +CALLBACK_URI = 'oob' - BASE_URL = "http://query.yahooapis.com/v1/yql" - REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token" - ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token" - AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth?oauth_token=" - CALLBACK_URI = 'oob' +class OAuth(object): + def __init__(self, ck=None, cs=None): """ """ self.ck = credentials.ck self.cs = credentials.cs - self.oauth = OAuth1(self.ck, client_secret=self.cs, callback_uri=self.CALLBACK_URI) + self.oauth = OAuth1(self.ck, client_secret=self.cs, callback_uri=CALLBACK_URI) - def request_token(self,): + def fetch_tokens(self, content, key, secret): + """Parse content to fetch request/access token/token-secret """ + stuff = parse_qs(content) + return stuff.get(key)[0], stuff.get(secret)[0] + + def request_token(self,): + """Get request token """ - response = requests.post(url=self.REQUEST_TOKEN_URL, auth=self.oauth) - credentias = parse_qs(response.content) - self.request_token = credentias.get('oauth_token')[0] - self.request_token_secret = credentias.get('oauth_token_secret')[0] + response = requests.post(url=REQUEST_TOKEN_URL, auth=self.oauth) + #credentials = parse_qs(response.content) + #self.request_token = credentials.get('oauth_token')[0] + #self.request_token_secret = credentials.get('oauth_token_secret')[0] + self.request_token, self.request_token_secret = self.fetch_tokens(response.content, 'oauth_token', 'oauth_token_secret') print(self.request_token, self.request_token_secret) return self.request_token, self.request_token_secret def get_user_authorization(self,): + """Get authorization """ - """ - webbrowser.open(self.AUTHORIZE_TOKEN_URL+self.request_token) + authorization_url = AUTHORIZE_TOKEN_URL+self.request_token + print(authorization_url) + webbrowser.open(authorization_url) self.verifier = raw_input("Please input a verifier: ") print(self.verifier) + def get_access_token(self): + """Get access token + """ + self.oauth = OAuth1(self.ck, client_secret=self.cs, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) + response = requests.post(url=ACCESS_TOKEN_URL, auth=self.oauth) + self.access_token, self.access_token_secret = self.fetch_tokens(response.content, 'oauth_token', 'oauth_token_secret') + print(self.access_token, self.access_token_secret) + return self.access_token, self.access_token_secret + if '__main__' == __name__: auth = OAuth() auth.request_token() auth.get_user_authorization() + auth.get_access_token() From 9d873aa5a29e9dbe475436c15d65f195a344cca6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 16:04:28 +0200 Subject: [PATCH 194/582] #12 : fetch tokens method improved --- myql/contrib/auth/auth.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 318eedc..3f1751a 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -22,11 +22,11 @@ def __init__(self, ck=None, cs=None): self.cs = credentials.cs self.oauth = OAuth1(self.ck, client_secret=self.cs, callback_uri=CALLBACK_URI) - def fetch_tokens(self, content, key, secret): + def fetch_tokens(self, content): """Parse content to fetch request/access token/token-secret """ stuff = parse_qs(content) - return stuff.get(key)[0], stuff.get(secret)[0] + return stuff.get('oauth_token')[0], stuff.get('oauth_token_secret')[0] def request_token(self,): """Get request token @@ -35,7 +35,7 @@ def request_token(self,): #credentials = parse_qs(response.content) #self.request_token = credentials.get('oauth_token')[0] #self.request_token_secret = credentials.get('oauth_token_secret')[0] - self.request_token, self.request_token_secret = self.fetch_tokens(response.content, 'oauth_token', 'oauth_token_secret') + self.request_token, self.request_token_secret = self.fetch_tokens(response.content) print(self.request_token, self.request_token_secret) return self.request_token, self.request_token_secret @@ -54,7 +54,7 @@ def get_access_token(self): """ self.oauth = OAuth1(self.ck, client_secret=self.cs, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) response = requests.post(url=ACCESS_TOKEN_URL, auth=self.oauth) - self.access_token, self.access_token_secret = self.fetch_tokens(response.content, 'oauth_token', 'oauth_token_secret') + self.access_token, self.access_token_secret = self.fetch_tokens(response.content) print(self.access_token, self.access_token_secret) return self.access_token, self.access_token_secret From c04ca235005c64d02b37ce3923baaa3b8291f86d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 16:08:50 +0200 Subject: [PATCH 195/582] json file config handler ok --- myql/contrib/auth/auth.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 3f1751a..8f2aa79 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -1,4 +1,5 @@ import pdb +import json import time import requests from requests_oauthlib import OAuth1 @@ -28,13 +29,27 @@ def fetch_tokens(self, content): stuff = parse_qs(content) return stuff.get('oauth_token')[0], stuff.get('oauth_token_secret')[0] + def json_get_data(self, filename): + """Returns content of a json file + """ + with open(filename) as f: + json_data = json.load(filename) + + return json_data + + def json_wirte_data(self, json_data, filename): + """Write data into a json file + """ + with open(filename, 'w') as f: + json.dump(json_data, f) + return True + + return False + def request_token(self,): """Get request token """ response = requests.post(url=REQUEST_TOKEN_URL, auth=self.oauth) - #credentials = parse_qs(response.content) - #self.request_token = credentials.get('oauth_token')[0] - #self.request_token_secret = credentials.get('oauth_token_secret')[0] self.request_token, self.request_token_secret = self.fetch_tokens(response.content) print(self.request_token, self.request_token_secret) return self.request_token, self.request_token_secret From b89297c7aff377eed6a34cfdc737a3be3f90aeef Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 16:17:31 +0200 Subject: [PATCH 196/582] #12 : old class using rauth deleted --- myql/contrib/auth/__init__.py | 4 +- myql/contrib/auth/yahooauth.py | 84 ---------------------------------- 2 files changed, 2 insertions(+), 86 deletions(-) delete mode 100755 myql/contrib/auth/yahooauth.py diff --git a/myql/contrib/auth/__init__.py b/myql/contrib/auth/__init__.py index 6959faf..0e8038c 100755 --- a/myql/contrib/auth/__init__.py +++ b/myql/contrib/auth/__init__.py @@ -4,8 +4,8 @@ # Filename : __init__.py # Description : # Creation Date : 03-03-2015 -# Last Modified : Tue 03 Mar 2015 06:24:16 AM CST +# Last Modified : Mon 20 Apr 2015 04:16:24 PM CEST # ################################################## -from yahooauth import YahooOAuth +from auth import OAuth diff --git a/myql/contrib/auth/yahooauth.py b/myql/contrib/auth/yahooauth.py deleted file mode 100755 index fd4e18a..0000000 --- a/myql/contrib/auth/yahooauth.py +++ /dev/null @@ -1,84 +0,0 @@ -import time -import webbrowser -from requests_oauthlib import OAuth1 - -class YahooOAuth(object): - '''OAuth for yahoo api - ''' - - default_base_url = 'http://query.yahooapis.com/v1/yql' - - def __init__(self, consumer_key= None, consumer_secret= None, base_url= default_base_url): - ''' - ''' - self.consumer_key = consumer_key - self.consumer_secret = consumer_secret - self.base_url = base_url - self.service = OAuth1Service( - consumer_key = consumer_key, - consumer_secret = consumer_secret, - name = 'yahoo', - request_token_url = "https://api.login.yahoo.com/oauth/v2/get_request_token", - access_token_url = "https://api.login.yahoo.com/oauth/v2/get_token", - authorize_url = "https://api.login.yahoo.com/oauth/v2/request_auth", - base_url = base_url - ) - - self.params = { - 'oauth_version' : 1.0, - 'oauth_callback' : 'oob' - } - - - def get_request_token(self,): - '''Requests access_token and access secrets - ''' - - self.request_token, self.request_token_secret = self.service.get_request_token(params= self.params) - - return self.request_token, self.request_token_secret - - def get_user_authorization(self,): - '''Redirects to the authorization url - ''' - - url = self.service.get_authorize_url(self.request_token) - - webbrowser.open(str(url)) - - self.verifier = raw_input('Please input the verifier : ') - - return self.verifier - - def get_access_token(self,): - '''Get oauth_token - ''' - - self.access_token, self.access_token_secret = self.service.get_access_token(self.request_token, self.request_token_secret, params={'oauth_verifier': self.verifier}) - - return self.access_token, self.access_token_secret - - def get_session(self, method='GET', **kwargs): - '''Returns a session object - ''' - self.session = self.service.get_auth_session(self.request_token, self.request_token_secret) - - return self.session - - #Left to be done - def refresh_token(self): - '''Refresh the access_token - ''' - pass - - #Not finsished yet - def authenticate(self,): - '''Handles the authentication process - ''' - self.get_request_token() - self.get_user_authorization() - self.get_access_token() - - self.payload = { - 'realm': "yahoo.apis.com", - } From 78fec04bb04950730b12404317344ab7a7205ba6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 16:29:01 +0200 Subject: [PATCH 197/582] #12 : TestOAuth added --- tests/__init__.py | 2 +- tests/tests.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 031965b..ecadf74 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1 @@ -from tests import TestTable +from tests import * diff --git a/tests/tests.py b/tests/tests.py index 21c2232..1b96dee 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -3,6 +3,9 @@ import unittest from xml.dom import minidom from xml.etree import cElementTree as xtree + +from myql.contrib.auth import OAuth + from myql.contrib.table import Table from myql.contrib.table import Base, BaseInput from myql.contrib.table import Binder, BinderFunction, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset @@ -13,6 +16,15 @@ logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logging.getLogger(__name__) + +class TestOAuth(unittest.TestCase): + + def setUp(self,): + pass + + def tearUp(self,): + pass + class TestTable(unittest.TestCase): def setUp(self,): From c0cd3f0ec3e53b0b6547093e0adc048d5b45deaf Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 16:30:41 +0200 Subject: [PATCH 198/582] #12 : yahooauth changed ino auth --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index ff41911..6cc7b1e 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -1,5 +1,5 @@ import requests -from contrib import yahooauth +from contrib.auth import OAuth import errors import importlib From a587ab032b823c5ab3c3f82ee5704ce8e06bdda4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 16:39:13 +0200 Subject: [PATCH 199/582] removing useless stuff --- myql/contrib/auth/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/myql/contrib/auth/__init__.py b/myql/contrib/auth/__init__.py index 0e8038c..05d034a 100755 --- a/myql/contrib/auth/__init__.py +++ b/myql/contrib/auth/__init__.py @@ -1,11 +1 @@ -################################################## -# -# Author : yosuke -# Filename : __init__.py -# Description : -# Creation Date : 03-03-2015 -# Last Modified : Mon 20 Apr 2015 04:16:24 PM CEST -# -################################################## - from auth import OAuth From dbf3349f655108c83c363d0f5fd094cde40ada35 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 17:08:12 +0200 Subject: [PATCH 200/582] #12 : fix failed builds --- myql/contrib/auth/auth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 8f2aa79..c8fdafd 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -16,11 +16,11 @@ class OAuth(object): - def __init__(self, ck=None, cs=None): + def __init__(self, consumer_key, consumer_secret,request_token=None, request_token_secret=None, verifier=None, access_token=None, access_token_secret=None, from_file=None): """ """ - self.ck = credentials.ck - self.cs = credentials.cs + self.ck = consumer_key + self.cs = consumer_secret self.oauth = OAuth1(self.ck, client_secret=self.cs, callback_uri=CALLBACK_URI) def fetch_tokens(self, content): From 844a7a24d5a3081d3e35144245ebb061462f2d3e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 20 Apr 2015 17:34:28 +0200 Subject: [PATCH 201/582] #12 : fxing failed build --- myql/contrib/auth/auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index c8fdafd..00cf96a 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -5,7 +5,6 @@ from requests_oauthlib import OAuth1 from urlparse import parse_qs import webbrowser -import credentials BASE_URL = "http://query.yahooapis.com/v1/yql" REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token" From 27a9d91c101f01004adc836bd572dcf4cfe8e371 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 21 Apr 2015 15:39:23 +0200 Subject: [PATCH 202/582] #12 : OAuth process kind of OK --- myql/contrib/auth/auth.py | 66 +++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 00cf96a..51f136a 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -13,14 +13,42 @@ CALLBACK_URI = 'oob' class OAuth(object): - - - def __init__(self, consumer_key, consumer_secret,request_token=None, request_token_secret=None, verifier=None, access_token=None, access_token_secret=None, from_file=None): + """ + """ + def __init__(self, consumer_key, consumer_secret, **kwargs): """ """ - self.ck = consumer_key - self.cs = consumer_secret - self.oauth = OAuth1(self.ck, client_secret=self.cs, callback_uri=CALLBACK_URI) + if kwargs.get('from_file'): + from_file = kwargs.get('from_file') + json_data = self.json_get_data(from_file) + vars(self).update(json_data) + else: + self.consumer_key = consumer_key + self.consumer_secret = consumer_secret + vars(self).update(kwargs) + + if not self.access_token and not self.access_token_secret: + + if not self.request_token and not self.request_token_secret: + self.get_request_token() + self.verifier = self.get_user_authorization() + elif not self.verifier: + self.verifier = self.get_user_authorization() + self.get_access_token() + else: + self.get_access_token() + else: + self.oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.access_token, resource_owner_secret=self.access_token_secret) + + json_data.update({ + 'request_token': self.request_token, + 'request_token_secret': self.request_token_secret, + 'verifier': self.verifier, + 'access_token': self.access_token, + 'access_token_secret': self.access_token_secret + }) + + self.json_wirte_data(json_data, from_file) def fetch_tokens(self, content): """Parse content to fetch request/access token/token-secret @@ -31,24 +59,25 @@ def fetch_tokens(self, content): def json_get_data(self, filename): """Returns content of a json file """ - with open(filename) as f: - json_data = json.load(filename) + with open(filename) as fp: + json_data = json.load(fp) return json_data def json_wirte_data(self, json_data, filename): """Write data into a json file """ - with open(filename, 'w') as f: - json.dump(json_data, f) + with open(filename, 'w') as fp: + json.dump(json_data, fp, indent=4, encoding= 'utf-8', sort_keys=True) return True return False - def request_token(self,): + def get_request_token(self,): """Get request token """ - response = requests.post(url=REQUEST_TOKEN_URL, auth=self.oauth) + oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, callback_uri=CALLBACK_URI) + response = requests.post(url=REQUEST_TOKEN_URL, auth=oauth) self.request_token, self.request_token_secret = self.fetch_tokens(response.content) print(self.request_token, self.request_token_secret) return self.request_token, self.request_token_secret @@ -59,23 +88,20 @@ def get_user_authorization(self,): authorization_url = AUTHORIZE_TOKEN_URL+self.request_token print(authorization_url) webbrowser.open(authorization_url) - self.verifier = raw_input("Please input a verifier: ") + verifier = raw_input("Please input a verifier: ") print(self.verifier) - + return verifier def get_access_token(self): """Get access token """ - self.oauth = OAuth1(self.ck, client_secret=self.cs, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) - response = requests.post(url=ACCESS_TOKEN_URL, auth=self.oauth) + oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) + response = requests.post(url=ACCESS_TOKEN_URL, auth=oauth) self.access_token, self.access_token_secret = self.fetch_tokens(response.content) print(self.access_token, self.access_token_secret) return self.access_token, self.access_token_secret if '__main__' == __name__: - auth = OAuth() - auth.request_token() - auth.get_user_authorization() - auth.get_access_token() + auth = OAuth(None, None, from_file='credentials.json') From f15ccbe9b9d0343e4f3348752db3c916df09e3e7 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 21 Apr 2015 15:44:59 +0200 Subject: [PATCH 203/582] cleaning the cat house --- myql/myql.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 6cc7b1e..456242f 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -113,24 +113,6 @@ def buildResponse(self, response): return response - # def loadConfig(self, module): - # '''Loads OAuth config (consumer_key, consumer_secret) from module - # ''' - # try: - # config_module = importlib.import_module(module) - # except Exception, e: - # raise errors.NoConfigFileError(e) - - # try: - # ck = config_module.consumer_key - # cs = config_module.consumer_secret - # except Exception, e: - # raise errors.NoConfigParameter(e) - - # self.yoauth = yahooauth.YahooOAuth(ck, cs) - - # return config_module - ###################################################### # # MAIN METHODS From 7e87a1f30248c5de3b5f4feb664f8123c224768f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 21 Apr 2015 16:12:00 +0200 Subject: [PATCH 204/582] #12 : OAuth hanlder added --- myql/myql.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 456242f..3aded31 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -16,11 +16,11 @@ class MYQL(object): - diagnostics : set to to see diagnostics on queries - community : set to to have access to community tables ''' - default_url = 'https://query.yahooapis.com/v1/public/yql' - oauth_url = 'http://query.yahooapis.com/v1/yql' + public_url = 'https://query.yahooapis.com/v1/public/yql' + private_url = 'http://query.yahooapis.com/v1/yql' community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table - def __init__(self, table=None, url=default_url, community=False, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): + def __init__(self, table=None, url=public_url, community=False, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): self.url = url self.table = table self.format = format @@ -33,6 +33,9 @@ def __init__(self, table=None, url=default_url, community=False, format='json', self.crossProduct = crossProduct self.jsonCompact = jsonCompact self.debug = debug + + if oauth: + self.oauth = oauth.oauth def __repr__(self): '''Returns information on the current instance @@ -46,7 +49,7 @@ def payloadBuilder(self, query, format='json'): payload = { 'q' : query, - 'callback' : '', #This is not javascript + 'callback' : 'oop', #This is not javascript 'diagnostics' : self.diagnostics, 'format' : format, 'debug': self.debug, @@ -79,8 +82,11 @@ def rawQuery(self, query, format='', pretty=False): def executeQuery(self, payload): '''Execute the query and returns and response''' - - response = requests.get(self.url, params= payload) + if self.oauth: + self.url = self.private_url + response = requests.get(self.url, params= payload, auth= self.oauth) + else: + response = requests.get(self.url, params= payload) return response From c85ac8c3accbd83d7ab5f70f731e088d18c3bfc1 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 21 Apr 2015 16:12:58 +0200 Subject: [PATCH 205/582] #12 :cleaning the auth house --- myql/contrib/auth/auth.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 51f136a..c03800c 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -101,7 +101,4 @@ def get_access_token(self): print(self.access_token, self.access_token_secret) return self.access_token, self.access_token_secret -if '__main__' == __name__: - - auth = OAuth(None, None, from_file='credentials.json') From ed3b26bad239e455f9abe3326485824f041334c5 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 21 Apr 2015 17:29:21 +0200 Subject: [PATCH 206/582] #12 : now get access token when request tokens don't exist --- myql/contrib/auth/auth.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index c03800c..dd7d49b 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -32,6 +32,7 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): if not self.request_token and not self.request_token_secret: self.get_request_token() self.verifier = self.get_user_authorization() + self.get_access_token() elif not self.verifier: self.verifier = self.get_user_authorization() self.get_access_token() @@ -49,6 +50,11 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): }) self.json_wirte_data(json_data, from_file) + + def refresh_token(self): + """Refresh access token + """ + pass def fetch_tokens(self, content): """Parse content to fetch request/access token/token-secret @@ -97,8 +103,11 @@ def get_access_token(self): """ oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) response = requests.post(url=ACCESS_TOKEN_URL, auth=oauth) + import pdb + pdb.set_trace() self.access_token, self.access_token_secret = self.fetch_tokens(response.content) print(self.access_token, self.access_token_secret) return self.access_token, self.access_token_secret - +if '__main__' == __name__: + auth = OAuth(None, None, from_file='credentials.json') From 15076e0c30245d28dc761e3e0e8d1ce51a171592 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 21 Apr 2015 17:29:56 +0200 Subject: [PATCH 207/582] OAuth test updated --- tests/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 1b96dee..1988ae4 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -4,6 +4,7 @@ from xml.dom import minidom from xml.etree import cElementTree as xtree +from myql import MYQL from myql.contrib.auth import OAuth from myql.contrib.table import Table @@ -25,6 +26,12 @@ def setUp(self,): def tearUp(self,): pass + def test_get_guid(self,): + oauth = OAuth(None, None, from_file='credentials.json') + yql = MYQL(format='json', oauth=oauth) + response = yql.getGUID('josue_brunel') + pdb.set_trace() + class TestTable(unittest.TestCase): def setUp(self,): From 27a9b1aeb27012aa5d585ad83a9da684529be02f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 22 Apr 2015 17:46:31 +0200 Subject: [PATCH 208/582] removing pdb --- myql/contrib/auth/auth.py | 2 -- tests/tests.py | 1 - 2 files changed, 3 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index dd7d49b..c36ebd1 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -103,8 +103,6 @@ def get_access_token(self): """ oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) response = requests.post(url=ACCESS_TOKEN_URL, auth=oauth) - import pdb - pdb.set_trace() self.access_token, self.access_token_secret = self.fetch_tokens(response.content) print(self.access_token, self.access_token_secret) return self.access_token, self.access_token_secret diff --git a/tests/tests.py b/tests/tests.py index 1988ae4..a231219 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -30,7 +30,6 @@ def test_get_guid(self,): oauth = OAuth(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) response = yql.getGUID('josue_brunel') - pdb.set_trace() class TestTable(unittest.TestCase): From fc47e4a9d2a3d75f962d93c57cb9d7a6c0c865e9 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 23 Apr 2015 11:47:18 +0200 Subject: [PATCH 209/582] #12: oauth_session_handle fetched --- myql/contrib/auth/auth.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index c36ebd1..d4815d3 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -46,10 +46,17 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): 'request_token_secret': self.request_token_secret, 'verifier': self.verifier, 'access_token': self.access_token, - 'access_token_secret': self.access_token_secret + 'access_token_secret': self.access_token_secret, + 'session_hanlde': self.session_handle, + 'token_time': time.time() }) self.json_wirte_data(json_data, from_file) + + def isTokenValid(self): + """Check if the token hasn't expired + """ + pass def refresh_token(self): """Refresh access token @@ -60,7 +67,8 @@ def fetch_tokens(self, content): """Parse content to fetch request/access token/token-secret """ stuff = parse_qs(content) - return stuff.get('oauth_token')[0], stuff.get('oauth_token_secret')[0] + stuff = {k:v[0] for (k,v) in stuff.items()} + return stuff def json_get_data(self, filename): """Returns content of a json file @@ -84,9 +92,10 @@ def get_request_token(self,): """ oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, callback_uri=CALLBACK_URI) response = requests.post(url=REQUEST_TOKEN_URL, auth=oauth) - self.request_token, self.request_token_secret = self.fetch_tokens(response.content) + tokens = self.fetch_tokens(response.content) + self.request_token, self.request_token_secret = tokens.get('oauth_token'), tokens.get('oauth_token_secret') print(self.request_token, self.request_token_secret) - return self.request_token, self.request_token_secret + return tokens def get_user_authorization(self,): """Get authorization @@ -103,9 +112,10 @@ def get_access_token(self): """ oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) response = requests.post(url=ACCESS_TOKEN_URL, auth=oauth) - self.access_token, self.access_token_secret = self.fetch_tokens(response.content) + tokens = self.fetch_tokens(response.content) + self.access_token, self.access_token_secret, self.session_handle = tokens.get('oauth_token'), tokens.get('oauth_token_access'), tokens.get('oauth_session_handle',None) print(self.access_token, self.access_token_secret) - return self.access_token, self.access_token_secret + return tokens if '__main__' == __name__: auth = OAuth(None, None, from_file='credentials.json') From 15bb1c4fe6f07476a539273e8ff8977c3be23dd0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 23 Apr 2015 11:57:31 +0200 Subject: [PATCH 210/582] #12 : fix issue with access_token_secret recording --- myql/contrib/auth/auth.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index d4815d3..61288f7 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -27,13 +27,13 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): self.consumer_secret = consumer_secret vars(self).update(kwargs) - if not self.access_token and not self.access_token_secret: + if not vars(self).get('access_token') and not vars(self).get('access_token_secret'): - if not self.request_token and not self.request_token_secret: + if not vars(self).get('request_token') and not vars(self).get('request_token_secret'): self.get_request_token() self.verifier = self.get_user_authorization() self.get_access_token() - elif not self.verifier: + elif not vars(self).get('verifier'): self.verifier = self.get_user_authorization() self.get_access_token() else: @@ -93,6 +93,7 @@ def get_request_token(self,): oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, callback_uri=CALLBACK_URI) response = requests.post(url=REQUEST_TOKEN_URL, auth=oauth) tokens = self.fetch_tokens(response.content) + print(tokens) self.request_token, self.request_token_secret = tokens.get('oauth_token'), tokens.get('oauth_token_secret') print(self.request_token, self.request_token_secret) return tokens @@ -104,7 +105,7 @@ def get_user_authorization(self,): print(authorization_url) webbrowser.open(authorization_url) verifier = raw_input("Please input a verifier: ") - print(self.verifier) + print(verifier) return verifier def get_access_token(self): @@ -113,7 +114,8 @@ def get_access_token(self): oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) response = requests.post(url=ACCESS_TOKEN_URL, auth=oauth) tokens = self.fetch_tokens(response.content) - self.access_token, self.access_token_secret, self.session_handle = tokens.get('oauth_token'), tokens.get('oauth_token_access'), tokens.get('oauth_session_handle',None) + print(tokens) + self.access_token, self.access_token_secret, self.session_handle = tokens.get('oauth_token'), tokens.get('oauth_token_secret'), tokens.get('oauth_session_handle') print(self.access_token, self.access_token_secret) return tokens From ce2a92c06dda60ef69763acf6a80173a9f474024 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 23 Apr 2015 12:09:08 +0200 Subject: [PATCH 211/582] #12 : trying to fix the readme --- myql/contrib/auth/readme.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/myql/contrib/auth/readme.md b/myql/contrib/auth/readme.md index 6d2121b..0e8ad1f 100755 --- a/myql/contrib/auth/readme.md +++ b/myql/contrib/auth/readme.md @@ -1,28 +1,28 @@ -YahooOAuth +OAuth ========== -***YahooOAuth*** is 3 legged *OAuth* module dedicated to the *Yahoo! OAuth Provider*. It uses [rauth](http://rauth.readthedocs.org/en/latest/) as *oauth* library. +***Auth*** is 3 legged *OAuth* module dedicated to the *Yahoo! OAuth Provider*. It uses [rauth](http://rauth.readthedocs.org/en/latest/) as *oauth* library. You only need **2** parameters to make it work : - Your ***consumer key*** - Your ***consumer secret*** I tried to make it as painless as possible for me (and you too ;) ). Thus **3 legged OAuth** , **3 methods** and **3 steps** ( in order ) : -* YahooOAuth.get_request_token() -* YahooOAuth.get_user_authorization() -* YahooOAuth.get_access_token() +* Auth.get_request_token() +* Auth.get_user_authorization() +* Auth.get_access_token() Quick start ----------- ```python ->>> from lokingyql import YahooOAuth ->>> from lokingyql.config import consumer_key, consumer_secret # config file where you have your consumer_key and consumer_secret +>>> from myql import OAuth +>>> from myql.config import consumer_key, consumer_secret # config file where you have your consumer_key and consumer_secret >>> consumer_key 'dj0yJmk9aVRSd3ZabElmTzJNJmQ9WVdrOWEyNW1VRmRGTnpZbWNHbzlNQS0tJnM9Y29uc3VtZXJzZWNyZXQmeD1hMg--' >>> consumer_secret '***************************' # Sorry guys it's kinda personal ->>> auth = YahooOAuth(consumer_key, consumer_secret) +>>> auth = Auth(consumer_key, consumer_secret) >>> #Step 1:Getting request token ... >>> auth.get_request_token() From 05637bf33bcc407d3e8799a34b3125d78abd1589 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 23 Apr 2015 12:26:09 +0200 Subject: [PATCH 212/582] #12 : refresh token method added --- myql/contrib/auth/auth.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 61288f7..744333f 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -1,6 +1,7 @@ import pdb import json import time +import logging import requests from requests_oauthlib import OAuth1 from urlparse import parse_qs @@ -47,21 +48,28 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): 'verifier': self.verifier, 'access_token': self.access_token, 'access_token_secret': self.access_token_secret, - 'session_hanlde': self.session_handle, + 'session_handle': self.session_handle, 'token_time': time.time() }) self.json_wirte_data(json_data, from_file) - def isTokenValid(self): + def isStillValid(self): """Check if the token hasn't expired """ - pass + elapsed_time = time.time() - vars(self).get('token_time') + if elapsed_time > 3000 : + return False + return True def refresh_token(self): """Refresh access token """ - pass + oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token, oauth_session_handle= self.session_handle) + response = requests.post(REQUEST_TOKEN_URL, auth=oauth) + tokens = self.fetch_tokens(response.content) + #self.oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.access_token, resource_owner_secret=self.access_token_secret) + return tokens def fetch_tokens(self, content): """Parse content to fetch request/access token/token-secret From f735c3bf45b4dd97caeb0050fda5fba6c598993c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 23 Apr 2015 14:00:09 +0200 Subject: [PATCH 213/582] #59 : fix logging in auth module --- myql/contrib/auth/auth.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 744333f..88cc47f 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -13,6 +13,9 @@ AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth?oauth_token=" CALLBACK_URI = 'oob' +logging.basicConfig(level=logging.DEBUG, format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") +logging.getLogger(__name__) + class OAuth(object): """ """ @@ -57,8 +60,11 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): def isStillValid(self): """Check if the token hasn't expired """ - elapsed_time = time.time() - vars(self).get('token_time') - if elapsed_time > 3000 : + elapsed_time = time.time() - vars(self).get('token_time',0) + if elapsed_time > 3000 and elapsed_time < 3600: + self.refresh_token() + True + elif elapsed_time > 3600: return False return True @@ -101,19 +107,19 @@ def get_request_token(self,): oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, callback_uri=CALLBACK_URI) response = requests.post(url=REQUEST_TOKEN_URL, auth=oauth) tokens = self.fetch_tokens(response.content) - print(tokens) + logging.debug(tokens) self.request_token, self.request_token_secret = tokens.get('oauth_token'), tokens.get('oauth_token_secret') - print(self.request_token, self.request_token_secret) + logging.debug("{0}, {1}".format(self.request_token, self.request_token_secret)) return tokens def get_user_authorization(self,): """Get authorization """ authorization_url = AUTHORIZE_TOKEN_URL+self.request_token - print(authorization_url) + logging.debug(authorization_url) webbrowser.open(authorization_url) verifier = raw_input("Please input a verifier: ") - print(verifier) + logging.debug(verifier) return verifier def get_access_token(self): @@ -122,9 +128,9 @@ def get_access_token(self): oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) response = requests.post(url=ACCESS_TOKEN_URL, auth=oauth) tokens = self.fetch_tokens(response.content) - print(tokens) + logging.debug(tokens) self.access_token, self.access_token_secret, self.session_handle = tokens.get('oauth_token'), tokens.get('oauth_token_secret'), tokens.get('oauth_session_handle') - print(self.access_token, self.access_token_secret) + logging.debug("{0} {1}".format(self.access_token, self.access_token_secret)) return tokens if '__main__' == __name__: From 55503921924b2b235ffd11701df01634c18dae12 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 23 Apr 2015 14:11:33 +0200 Subject: [PATCH 214/582] fix #12 : OAuth finally over :smiley: --- myql/contrib/auth/auth.py | 3 ++- myql/myql.py | 6 ++++-- tests/test_config.py | 2 -- tests/tests.py | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) delete mode 100755 tests/test_config.py diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 88cc47f..c007db6 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -57,12 +57,13 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): self.json_wirte_data(json_data, from_file) - def isStillValid(self): + def isValid(self): """Check if the token hasn't expired """ elapsed_time = time.time() - vars(self).get('token_time',0) if elapsed_time > 3000 and elapsed_time < 3600: self.refresh_token() + logging.debug("Token Refreshed") True elif elapsed_time > 3600: return False diff --git a/myql/myql.py b/myql/myql.py index 3aded31..74f2a52 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -35,7 +35,7 @@ def __init__(self, table=None, url=public_url, community=False, format='json', j self.debug = debug if oauth: - self.oauth = oauth.oauth + self.oauth = oauth def __repr__(self): '''Returns information on the current instance @@ -84,7 +84,9 @@ def executeQuery(self, payload): '''Execute the query and returns and response''' if self.oauth: self.url = self.private_url - response = requests.get(self.url, params= payload, auth= self.oauth) + if not self.oauth.isValid(): + print("Reset your credentials : they don't seem to be valid anymore") + response = requests.get(self.url, params= payload, auth= self.oauth.oauth) else: response = requests.get(self.url, params= payload) diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100755 index 5389972..0000000 --- a/tests/test_config.py +++ /dev/null @@ -1,2 +0,0 @@ -consumer_key = 'josue' -consumer_secret = 'brunel' \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py index a231219..ba28ac0 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -30,6 +30,7 @@ def test_get_guid(self,): oauth = OAuth(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) response = yql.getGUID('josue_brunel') + logging.debug(response.content) class TestTable(unittest.TestCase): From 39fa148cc632e77519d120d7d265dff2bd73332c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 23 Apr 2015 15:56:22 +0200 Subject: [PATCH 215/582] Just setting some stuff up --- myql/contrib/auth/auth.py | 5 +++-- tests/tests.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index c007db6..c8b8f96 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -72,10 +72,10 @@ def isValid(self): def refresh_token(self): """Refresh access token """ - oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token, oauth_session_handle= self.session_handle) + oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token, callback_uri=CALLBACK_URI) + oauth.client.oauth_session_handle = self.session_handle response = requests.post(REQUEST_TOKEN_URL, auth=oauth) tokens = self.fetch_tokens(response.content) - #self.oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.access_token, resource_owner_secret=self.access_token_secret) return tokens def fetch_tokens(self, content): @@ -136,3 +136,4 @@ def get_access_token(self): if '__main__' == __name__: auth = OAuth(None, None, from_file='credentials.json') + auth.refresh_token() diff --git a/tests/tests.py b/tests/tests.py index ba28ac0..05071e6 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -30,7 +30,8 @@ def test_get_guid(self,): oauth = OAuth(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) response = yql.getGUID('josue_brunel') - logging.debug(response.content) + logging.debug(response.status_code,200) + class TestTable(unittest.TestCase): From 046109a42772f3cbfbbc3d3b6ad029a6f93658a6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 23 Apr 2015 17:15:28 +0200 Subject: [PATCH 216/582] fix #61 : should raise an exception anymore --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 74f2a52..c4cd250 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -82,7 +82,7 @@ def rawQuery(self, query, format='', pretty=False): def executeQuery(self, payload): '''Execute the query and returns and response''' - if self.oauth: + if vars(self).get('oauth'): self.url = self.private_url if not self.oauth.isValid(): print("Reset your credentials : they don't seem to be valid anymore") From cade05602aec974fba2c20d3c9b963d4f7301f1c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 24 Apr 2015 06:36:50 +0200 Subject: [PATCH 217/582] fix #62: oop removed from payloader --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index c4cd250..3b720ac 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -49,7 +49,7 @@ def payloadBuilder(self, query, format='json'): payload = { 'q' : query, - 'callback' : 'oop', #This is not javascript + 'callback' : '', #This is not javascript 'diagnostics' : self.diagnostics, 'format' : format, 'debug': self.debug, From 5f9cf39364b504a3db7374daec2fee61c08520c4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 25 Apr 2015 09:49:26 +0200 Subject: [PATCH 218/582] #12 : oauth_session_handle added to request header OK --- myql/contrib/auth/auth.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index c8b8f96..c205f25 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -72,9 +72,8 @@ def isValid(self): def refresh_token(self): """Refresh access token """ - oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token, callback_uri=CALLBACK_URI) - oauth.client.oauth_session_handle = self.session_handle - response = requests.post(REQUEST_TOKEN_URL, auth=oauth) + oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token,resource_owner_secret=self.access_token_secret, callback_uri=CALLBACK_URI) + response = requests.post(REQUEST_TOKEN_URL, headers={'oauth_session_handle': self.session_handle}, auth=oauth) tokens = self.fetch_tokens(response.content) return tokens From 552ca976a4a14019180c1c20d89d3ae9718f18b8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 25 Apr 2015 10:06:42 +0200 Subject: [PATCH 219/582] #1 Documentation folder Ok --- myql/contrib/auth/docs/docs/index.md | 17 +++++++++++++++++ myql/contrib/auth/docs/mkdocs.yml | 1 + 2 files changed, 18 insertions(+) create mode 100644 myql/contrib/auth/docs/docs/index.md create mode 100644 myql/contrib/auth/docs/mkdocs.yml diff --git a/myql/contrib/auth/docs/docs/index.md b/myql/contrib/auth/docs/docs/index.md new file mode 100644 index 0000000..da37213 --- /dev/null +++ b/myql/contrib/auth/docs/docs/index.md @@ -0,0 +1,17 @@ +# Welcome to MkDocs + +For full documentation visit [mkdocs.org](http://mkdocs.org). + +## Commands + +* `mkdocs new [dir-name]` - Create a new project. +* `mkdocs serve` - Start the live-reloading docs server. +* `mkdocs build` - Build the documentation site. +* `mkdocs help` - Print this help message. + +## Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. diff --git a/myql/contrib/auth/docs/mkdocs.yml b/myql/contrib/auth/docs/mkdocs.yml new file mode 100644 index 0000000..c97182f --- /dev/null +++ b/myql/contrib/auth/docs/mkdocs.yml @@ -0,0 +1 @@ +site_name: My Docs From 6c844fa4fa882719642c9f0809639c896e17d3a0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 25 Apr 2015 11:12:16 +0200 Subject: [PATCH 220/582] xml ignored --- .gitignore | 1 + {myql/contrib/auth/docs => docs}/docs/index.md | 0 {myql/contrib/auth/docs => docs}/mkdocs.yml | 0 3 files changed, 1 insertion(+) rename {myql/contrib/auth/docs => docs}/docs/index.md (100%) rename {myql/contrib/auth/docs => docs}/mkdocs.yml (100%) diff --git a/.gitignore b/.gitignore index e49de98..27b3734 100755 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ coverage.xml docs/_build/ myconfig.py +*.xml diff --git a/myql/contrib/auth/docs/docs/index.md b/docs/docs/index.md similarity index 100% rename from myql/contrib/auth/docs/docs/index.md rename to docs/docs/index.md diff --git a/myql/contrib/auth/docs/mkdocs.yml b/docs/mkdocs.yml similarity index 100% rename from myql/contrib/auth/docs/mkdocs.yml rename to docs/mkdocs.yml From 2d270ff30bc502dc5d00e9678db18f2c9e7dbdae Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 25 Apr 2015 11:14:10 +0200 Subject: [PATCH 221/582] #1 : Doc updated --- .gitignore | 1 + README.md | 27 +++----- docs/docs/contrib.md | 2 + docs/docs/index.md | 146 +++++++++++++++++++++++++++++++++++++++---- docs/docs/table.md | 1 + docs/mkdocs.yml | 7 ++- 6 files changed, 153 insertions(+), 31 deletions(-) create mode 100644 docs/docs/contrib.md create mode 100644 docs/docs/table.md diff --git a/.gitignore b/.gitignore index 27b3734..a2063ee 100755 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,4 @@ docs/_build/ myconfig.py *.xml +site diff --git a/README.md b/README.md index aabceac..50bf978 100755 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ MYQL MYQL is a Python wrapper of the Yahoo Query Language. -Yahoo! Query Langauge Documentation and Support +Yahoo! Query Language Documentation and Support =============================================== * Yahoo! Query Language - http://developer.yahoo.com/yql/ @@ -50,8 +50,7 @@ how to use >>> yql.diagnostics = True # To turn diagnostics on ``` -access to community tables --------------------------- +####access to community tables ```python >>> yql = myql.MYQL() @@ -73,8 +72,7 @@ access to community tables >>> # do your magic ``` -changing response format (xml or json) --------------------------------------- +####changing response format (xml or json) The response format is by default ***json***. @@ -94,18 +92,15 @@ u'\n>> +``` +or + +```python +>>> import myql +>>> yql = myql.MYQL(community=True) +>>> # do your magic +``` + +####Changing response format (xml or json) + +The response format is by default ***json***. + +```python +>>> import myql +>>> yql = myql.MYQL(format='xml', community=True) +>>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') +>>> rep.text +u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' +>>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') +>>> rep.json() +{u'query': {u'count': 11, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'55998384', u'name': u'Cuvette-Ouest Department'}, {u'woeid': u'2344968', u'name': u'Cuvette Department'}, {u'woeid': u'2344973', u'name': u'Plateaux District'}, {u'woeid': u'2344974', u'name': u'Sangha'}, {u'woeid': u'2344970', u'name': u'Lekoumou'}, {u'woeid': u'2344975', u'name': u'Pool Department'}, {u'woeid': u'2344971', u'name': u'Likouala Department'}, {u'woeid': u'2344972', u'name': u'Niari Department'}, {u'woeid': u'2344976', u'name': u'Brazzaville'}, {u'woeid': u'2344967', u'name': u'Bouenza Department'}, {u'woeid': u'2344969', u'name': u'Kouilou'}]}, u'created': u'2014-08-27T04:52:38Z'}} +>>> +``` + + +Methods +------- + +####use(data_provider_url) +Changes the data provider + +```python +>>> yql.use('http://myserver.com/mytables.xml') +``` + +####desc(tablename) +Returns table description + +```python +>>> response = yql.desc('weather.forecast') +>>> response.json() +{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'request': {u'select': [{u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'location'}, {u'type': u'xs:string', u'name': u'u'}]}, {u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'woeid'}, {u'type': u'xs:string', u'name': u'u'}]}]}, u'security': u'ANY', u'meta': {u'documentationURL': u'http://developer.yahoo.com/weather/', u'sampleQuery': u'select * from weather.forecast where woeid=2502265', u'description': u'Weather forecast table', u'author': u'Yahoo! Inc'}, u'hash': u'aae78b1462a6a8fbc748aec4cf292767', u'name': u'weather.forecast'}}, u'created': u'2014-08-16T19:31:51Z'}} +>>> +``` + +####rawQuery(query) + +Allows you to directly type your query + +```python +>>> response = yql.rawQuery("select * from geo.countries where place='North America'") +>>> # deal with the response +``` + +####select(table, fields, limit).where(filters, ...) + +Select a table i.e *weather.forecast*. +If *table* not provided, it will use the default table. If there's no such thing as a default table, it will raise a *NoTableSelectedError* + +***NB*** : A simple select doesn't return any data. Use ***GET*** instead. + +```python +>>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) +>>> response.json() +{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424775', u'name': u'Canada'}}, u'created': u'2014-08-16T19:04:08Z'}} +>>> ... +>>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', '=', 'Africa']) +>>> rep.json() +{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T10:52:49Z'}} +>>> +>>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) +>>> rep.json() +{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T11:22:49Z'}} +>>> +``` + +####get(table, fields, limit) +Same as ***SELECT***, but instead returns data. + +**REMINDER** : Some tables require a **where clause**, therefore ***GET*** won't work on those tables, use *select(...).where(...)* instead . + +```python +>>> yql.get('geo.countries', ['name', 'woeid'], 1) +>>> rep.json() +{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} +>>> +``` - mkdocs.yml # The configuration file. - docs/ - index.md # The documentation homepage. - ... # Other markdown pages, images and other files. diff --git a/docs/docs/table.md b/docs/docs/table.md new file mode 100644 index 0000000..ae8381b --- /dev/null +++ b/docs/docs/table.md @@ -0,0 +1 @@ +##OpenTable diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index c97182f..3bd74ac 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1 +1,6 @@ -site_name: My Docs +site_name: myql +pages: +- [index.md, Home] +- [table.md, OpenTable] +- [contrib.md, Contribute] +theme: readthedocs From 18463644583673ba13697bad8ba5e9c64734df61 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 25 Apr 2015 11:41:19 +0200 Subject: [PATCH 222/582] #1: updating doc for myq.rtfd.org --- docs/{docs => }/contrib.md | 0 docs/{docs => }/index.md | 0 docs/{docs => }/table.md | 0 docs/mkdocs.yml => mkdocs.yml | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename docs/{docs => }/contrib.md (100%) rename docs/{docs => }/index.md (100%) rename docs/{docs => }/table.md (100%) rename docs/mkdocs.yml => mkdocs.yml (100%) diff --git a/docs/docs/contrib.md b/docs/contrib.md similarity index 100% rename from docs/docs/contrib.md rename to docs/contrib.md diff --git a/docs/docs/index.md b/docs/index.md similarity index 100% rename from docs/docs/index.md rename to docs/index.md diff --git a/docs/docs/table.md b/docs/table.md similarity index 100% rename from docs/docs/table.md rename to docs/table.md diff --git a/docs/mkdocs.yml b/mkdocs.yml similarity index 100% rename from docs/mkdocs.yml rename to mkdocs.yml From f0f85ddc8da935094929a6b86d6498e36a848075 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 25 Apr 2015 19:12:20 +0200 Subject: [PATCH 223/582] #1 : Docs badge updated --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50bf978..20710a3 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ MYQL ========= -[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) +[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) ![Alt](https://readthedocs.org/projects/pip/badge/?version=latest) MYQL is a Python wrapper of the Yahoo Query Language. From 1a308c74e5e36d483de0201e6d94b31405d63510 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 26 Apr 2015 17:48:13 +0200 Subject: [PATCH 224/582] #12 : request_tokens and verifier not saved anymore --- myql/contrib/auth/auth.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index c205f25..52e9b47 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -32,23 +32,25 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): vars(self).update(kwargs) if not vars(self).get('access_token') and not vars(self).get('access_token_secret'): - - if not vars(self).get('request_token') and not vars(self).get('request_token_secret'): - self.get_request_token() - self.verifier = self.get_user_authorization() - self.get_access_token() - elif not vars(self).get('verifier'): - self.verifier = self.get_user_authorization() - self.get_access_token() - else: - self.get_access_token() - else: - self.oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.access_token, resource_owner_secret=self.access_token_secret) + self.get_request_token() + self.verifier = self.get_user_authorization() + self.get_access_token() + # if not vars(self).get('request_token') and not vars(self).get('request_token_secret'): + # self.get_request_token() + # self.verifier = self.get_user_authorization() + # self.get_access_token() + # elif not vars(self).get('verifier'): + # self.verifier = self.get_user_authorization() + # self.get_access_token() + # else: + # self.get_access_token() + #else: + self.oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.access_token, resource_owner_secret=self.access_token_secret) json_data.update({ - 'request_token': self.request_token, - 'request_token_secret': self.request_token_secret, - 'verifier': self.verifier, + #'request_token': self.request_token, + #'request_token_secret': self.request_token_secret, + #'verifier': self.verifier, 'access_token': self.access_token, 'access_token_secret': self.access_token_secret, 'session_handle': self.session_handle, @@ -72,8 +74,9 @@ def isValid(self): def refresh_token(self): """Refresh access token """ - oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token,resource_owner_secret=self.access_token_secret, callback_uri=CALLBACK_URI) - response = requests.post(REQUEST_TOKEN_URL, headers={'oauth_session_handle': self.session_handle}, auth=oauth) + oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token,resource_owner_secret=self.access_token_secret) + response = requests.post(ACCESS_TOKEN_URL, headers={'oauth_session_handle': self.session_handle}, auth=oauth) + pdb.set_trace() tokens = self.fetch_tokens(response.content) return tokens From 399cda63c04efa2cf9799627115f3af9c4a7e071 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 27 Apr 2015 13:44:17 +0200 Subject: [PATCH 225/582] fix rtfd badge issue --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 20710a3..38ac84e 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ MYQL ========= -[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) ![Alt](https://readthedocs.org/projects/pip/badge/?version=latest) +[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://readthedocs.org/projects/myql/?badge=latest) + MYQL is a Python wrapper of the Yahoo Query Language. From 24d58fb45537d85199474257354105c40fa0ecf8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 28 Apr 2015 12:38:38 +0200 Subject: [PATCH 226/582] Upper Name --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 3bd74ac..43f7214 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: myql +site_name: MYQL pages: - [index.md, Home] - [table.md, OpenTable] From 05bed3fd3d858c6fabfb3d6eda6bce30221488d6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 28 Apr 2015 14:23:11 +0200 Subject: [PATCH 227/582] #1 : OpenTable Table Doc OK --- docs/table.md | 42 +++++++++++++++++++++++++++++++- mkdocs.yml | 2 +- myql/contrib/table/exceptions.py | 1 + 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 myql/contrib/table/exceptions.py diff --git a/docs/table.md b/docs/table.md index ae8381b..192208b 100644 --- a/docs/table.md +++ b/docs/table.md @@ -1 +1,41 @@ -##OpenTable +YQL Open Table +============== + +## Table + +This class represents the **root** element of a YQL Table Definition File. You can read about the full documentation [here](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-tables-element) + +```python +def __init__(self, name, author, apiKeyURL, documentationURL, sampleQuery=[], description=None, table_attr=None, bindings=[]) +``` + +```python +>>> from myql.contrib.table import Table +>>> mytable = Table('mytable', 'Josue Kouka', 'http://josuerunel.org/mytable/','http://josuerunel.org/mytable/docs.html',sampleQuery = ['SELECT * FROM mytable', 'SELECT name FROM mytable WHERE id = 77'], description='Just a simple tabe', table_attr={'xmlns':'http://query.yahooapis.com/v1/schema/table.xsd', 'securityLevel':'any', 'https':'false'}) +``` + +####Table.addBinder(binder_object) +Add a binder to the table +```python +>>> mytable.addBinder(select_binder) +``` +####Table.removeBinder(binder_object) +Remove a binder from the table +```python +>>> mytable.removeBinder(select_binder) +``` +####Table.save(name=None, path=None) +Save the table as a *xml file* with Table Object name if *name* is not provided. If *path*, saves the *xml file* to the specified location +```python +>>> mytable.save(name='test',path='/var/www/josuebrunel.org/mytable/') +``` + +## Binder + +## Inputs + +## Paging + + + + diff --git a/mkdocs.yml b/mkdocs.yml index 3bd74ac..43f7214 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: myql +site_name: MYQL pages: - [index.md, Home] - [table.md, OpenTable] diff --git a/myql/contrib/table/exceptions.py b/myql/contrib/table/exceptions.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/myql/contrib/table/exceptions.py @@ -0,0 +1 @@ + From 00294cba08e28d46aac1877b0d117b34ac2f5e6e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 28 Apr 2015 14:42:23 +0200 Subject: [PATCH 228/582] #1 fix link to rtfd.org --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38ac84e..3902f80 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ MYQL ========= -[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://readthedocs.org/projects/myql/?badge=latest) +[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) MYQL is a Python wrapper of the Yahoo Query Language. From fdb05b4468cde55b1f251d662ec41a2542926919 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 28 Apr 2015 15:35:24 +0200 Subject: [PATCH 229/582] #1 : Table doc updated --- docs/table.md | 63 +++++++++++++++++++++++++++++++++++++++++++-------- mkdocs.yml | 2 +- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/docs/table.md b/docs/table.md index 192208b..945496e 100644 --- a/docs/table.md +++ b/docs/table.md @@ -2,40 +2,85 @@ YQL Open Table ============== ## Table - This class represents the **root** element of a YQL Table Definition File. You can read about the full documentation [here](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-tables-element) -```python -def __init__(self, name, author, apiKeyURL, documentationURL, sampleQuery=[], description=None, table_attr=None, bindings=[]) -``` +### **Definition** + +#### *Table(name, author, apiKeyURL, documentationURL, sampleQuery=[], description=None, table_attr=None, bindings=[])* ```python >>> from myql.contrib.table import Table >>> mytable = Table('mytable', 'Josue Kouka', 'http://josuerunel.org/mytable/','http://josuerunel.org/mytable/docs.html',sampleQuery = ['SELECT * FROM mytable', 'SELECT name FROM mytable WHERE id = 77'], description='Just a simple tabe', table_attr={'xmlns':'http://query.yahooapis.com/v1/schema/table.xsd', 'securityLevel':'any', 'https':'false'}) ``` -####Table.addBinder(binder_object) +### **Methods** + +#### *Table.addBinder(binder_object)* Add a binder to the table ```python >>> mytable.addBinder(select_binder) ``` -####Table.removeBinder(binder_object) +#### *Table.removeBinder(binder_object)* Remove a binder from the table ```python >>> mytable.removeBinder(select_binder) ``` -####Table.save(name=None, path=None) +#### *Table.save(name=None, path=None)* Save the table as a *xml file* with Table Object name if *name* is not provided. If *path*, saves the *xml file* to the specified location ```python >>> mytable.save(name='test',path='/var/www/josuebrunel.org/mytable/') ``` -## Binder - ## Inputs +There are 3 kind of *inputs* as described in the [documentation](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-key) : + +* _key_ representing by _InputKey_ +* _map_ representing by _InputMap_ +* _value_ representing by _InputValue_ + +### **Definition** + +### **Methods** ## Paging + +### **Definition** + +### **Methods** + +## Binder +This class represents an element under ****. Which means : + +* select +* insert +* update +* delete + +You can read about the full documentation [here](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-select) + +### **Definition** + +#### *Binder(name, itemPath, produces, pollingFrequencySeconds=0, urls=[], inputs=[], paging=None)* + +```python +>>> select = Binder('select', 'products.product', 'xml') +``` + +### **Methods** + +#### *Binder.addInput(input_object)* : +Add a input object to the binder +#### *Binder.removeInput(input_id, input_type)* +Remove an input object from the binder. ***input_type*** may be ***key, value or map*** +#### *Binder.addUrl(url)* +#### *Binder.removeUrl(url)* +#### *Binder.addPaging(paging_instance)* +#### *Binder.removePaging(paging_instance)* + +## MetaClasses +### **Definition** +### **Methods** diff --git a/mkdocs.yml b/mkdocs.yml index 43f7214..66cd5d6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,4 +3,4 @@ pages: - [index.md, Home] - [table.md, OpenTable] - [contrib.md, Contribute] -theme: readthedocs +theme: readthedocs \ No newline at end of file From 00a38f2a4aaad7f8976d7bc728cf490d50c6dd66 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 28 Apr 2015 15:48:55 +0200 Subject: [PATCH 230/582] #1 Inputs Doc OK --- docs/table.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/table.md b/docs/table.md index 945496e..e63cd28 100644 --- a/docs/table.md +++ b/docs/table.md @@ -34,13 +34,22 @@ Save the table as a *xml file* with Table Object name if *name* is not provided. ## Inputs There are 3 kind of *inputs* as described in the [documentation](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-key) : -* _key_ representing by _InputKey_ -* _map_ representing by _InputMap_ -* _value_ representing by _InputValue_ +* ***key*** +* ***map*** +* ***value*** -### **Definition** +### **Definitions** + +* #### *InputKey(id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=0)* + +* #### *InputValue(id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=0)* + +* #### *InputMap(id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=0)* + +All of those objects are based on ***BaseInput***. ### **Methods** +No methods defined ## Paging From f5485501d13467276ea27376bc7f048efc5f354c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 28 Apr 2015 16:01:00 +0200 Subject: [PATCH 231/582] #1 : Paging Doc OK --- docs/table.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/table.md b/docs/table.md index e63cd28..f5fcd66 100644 --- a/docs/table.md +++ b/docs/table.md @@ -52,10 +52,29 @@ All of those objects are based on ***BaseInput***. No methods defined ## Paging +This class describe a ***paging*** element. A ***paging*** is deifined by the 3 classes below : -### **Definition** +* ***PagingPage*** +* ***PagingUrl*** +* ***PagingOffset*** + +### **Definitions** + +* #### *PagingPage(start={}, pageSize={}, total={})* +```python +>>> mypage = PagingPage({'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) +``` +* #### *PagingUrl(nextpage)* +```python +>>> mypage = PagingUrl("ysearchresponse.nextpage") +``` +* #### *PagingOffset(matrix, start={}, pageSize={}, total={})* +```python +>>> mypage = PagingOffset({'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) +``` ### **Methods** +No methods defined ## Binder This class represents an element under ****. Which means : From eaebd11cd56e8f63e288f8d2b1737bcbfa96cfe6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 28 Apr 2015 16:32:45 +0200 Subject: [PATCH 232/582] #1 : Metaclass doc OK --- docs/index.md | 2 +- docs/table.md | 131 +++++++++++++++++++++++++++++++--- myql/contrib/table/example.py | 4 +- 3 files changed, 126 insertions(+), 11 deletions(-) diff --git a/docs/index.md b/docs/index.md index 4d10f37..4c7f589 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,7 +10,7 @@ Yahoo! Query Language Documentation and Support * Yahoo! Developer Network: http://developer.yahoo.com * Yahoo! Application Platform - http://developer.yahoo.com/yap/ * Yahoo! Social APIs - http://developer.yahoo.com/social/ -* Yahoo! QUery Language Console https://developer.yahoo.com/yql/console/ +* Yahoo! Query Language Console https://developer.yahoo.com/yql/console/ Installation ============ diff --git a/docs/table.md b/docs/table.md index f5fcd66..4ae7641 100644 --- a/docs/table.md +++ b/docs/table.md @@ -41,23 +41,32 @@ There are 3 kind of *inputs* as described in the [documentation](https://develop ### **Definitions** * #### *InputKey(id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=0)* - +```python +song = InputKey(id='song', type='xs:string', paramType='path', required=True) +``` * #### *InputValue(id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=0)* - +```python +song = InputValue(id='song', type='xs:string', paramType='path', required=True, const='12' ) +``` * #### *InputMap(id, type, paramType, like='', required=False, default='', private=False, const=False, batchable=False, maxBatchItems=0)* -All of those objects are based on ***BaseInput***. +All of those classes are based on ***BaseInput***. + +***like*** is a replacement for ***as*** which is a python keyword. In the *xml* file, ***as*** will be displayed. + + ### **Methods** No methods defined ## Paging -This class describe a ***paging*** element. A ***paging*** is deifined by the 3 classes below : +This class describe a ***paging*** element. A ***paging*** is deifined by one of the 3 classes below : * ***PagingPage*** * ***PagingUrl*** * ***PagingOffset*** +Check out the full [documentation](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-paging) ### **Definitions** * #### *PagingPage(start={}, pageSize={}, total={})* @@ -73,6 +82,8 @@ This class describe a ***paging*** element. A ***paging*** is deifined by the 3 >>> mypage = PagingOffset({'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) ``` +All these classes above subclass ***BasePaging*** . + ### **Methods** No methods defined @@ -83,6 +94,7 @@ This class represents an element under ****. Which means : * insert * update * delete +* function (stored function) You can read about the full documentation [here](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-select) @@ -97,18 +109,121 @@ You can read about the full documentation [here](https://developer.yahoo.com/yql ### **Methods** #### *Binder.addInput(input_object)* : -Add a input object to the binder +Add an input object to the binder #### *Binder.removeInput(input_id, input_type)* Remove an input object from the binder. ***input_type*** may be ***key, value or map*** #### *Binder.addUrl(url)* +Add an url to the binder #### *Binder.removeUrl(url)* +Remove an url from the binder #### *Binder.addPaging(paging_instance)* +Add a paging to the binder #### *Binder.removePaging(paging_instance)* +Remove a paging from the binder ## MetaClasses - -### **Definition** -### **Methods** +They say *"A picture is worth a thousand of word"* and I say *"A code snippet +is worth ..."* . You got it (^_^). +***BinderModel*** and ***TableModel*** are the only classes to use here. + +Copy and past the code snippet below in a *example.py* + +```python +from binder import BinderModel, InputKey, PagingPage, PagingUrl, InputValue, BinderFunction +from table import TableModel, BinderFrom + +class SelectBinder(BinderModel): + name = 'select' + itemPath = 'products.product' + produces = 'xml' + pollingFrequencySeconds = 30 + urls = ['http://lol.com/services?artist={artis}','http://lol.com/services/song={song}'] + paging = PagingPage({'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) + artist = InputKey(id='artist', type='xs:string', paramType='path') + song = InputKey(id='song', type='xs:string', paramType='path', required=True) + +class InsertBinder(BinderModel): + name = 'insert' + itemPath = 'products.product' + produces = 'xml' + pollingFrequencySeconds = 30 + urls = ['http://lol.com/services?artist={artis}','http://lol.com/services/song={song}'] + paging = PagingUrl(nextpage={'path':'yqlsearch.nextpage'}) + artist = InputKey(id='artist', type='xs:string', paramType='path') + song = InputValue(id='song', type='xs:string', paramType='path', required=True) + + +class TestTable(TableModel): + name = 'Test' + author = 'Josue Kouka' + apiKeyURL = 'http://josuebrunel.org/api' + documentationURL = 'http://josuebrunel.org/doc.html' + description = "Just a test table" + sampleQuery = ['SELECT * FROM mytable','SELECT name FROM mytable WHERE id=4656', "SELECT * FROM mytable WHERE name='Josh'"] + select = BinderFrom(SelectBinder) + insert = BinderFrom(InsertBinder) + func1 = BinderFunction('concat', func_code="console.log('Hello Josh!!!')") + +TestTable.table.save(name='Example') +``` +Run + +```shell +$ python example.py +$ cat Example.xml +``` + +```xml + + + + http://josuebrunel.org/api + Josue Kouka + Just a test table + http://josuebrunel.org/doc.html + SELECT * FROM mytable + SELECT name FROM mytable WHERE id=4656 + SELECT * FROM mytable WHERE name='Josh' + + + + + ![CDATA]console.log('Hello Josh!!!')]] + + + + + http://lol.com/services?artist={artis} + http://lol.com/services/song={song} + + + + + + + + + + + +
+ +``` +Voila, i think we're done here diff --git a/myql/contrib/table/example.py b/myql/contrib/table/example.py index eaeb193..2fbaa9a 100644 --- a/myql/contrib/table/example.py +++ b/myql/contrib/table/example.py @@ -9,7 +9,7 @@ class SelectBinder(BinderModel): urls = ['http://lol.com/services?artist={artis}','http://lol.com/services/song={song}'] paging = PagingPage({'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) artist = InputKey(id='artist', type='xs:string', paramType='path') - song = InputKey(id='song', type='xs:string', paramType='path', required='true') + song = InputKey(id='song', type='xs:string', paramType='path', required=True) class InsertBinder(BinderModel): name = 'insert' @@ -19,7 +19,7 @@ class InsertBinder(BinderModel): urls = ['http://lol.com/services?artist={artis}','http://lol.com/services/song={song}'] paging = PagingUrl(nextpage={'path':'yqlsearch.nextpage'}) artist = InputKey(id='artist', type='xs:string', paramType='path') - song = InputValue(id='song', type='xs:string', paramType='path', required='true') + song = InputValue(id='song', type='xs:string', paramType='path', required=True) class TestTable(TableModel): From 73fb7c7c62a0066de0aac18d4b852b0635e4bdda Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 28 Apr 2015 21:00:28 +0200 Subject: [PATCH 233/582] #1 : Doc updated --- docs/table.md | 60 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/table.md b/docs/table.md index 4ae7641..43cd65f 100644 --- a/docs/table.md +++ b/docs/table.md @@ -94,7 +94,7 @@ This class represents an element under ****. Which means : * insert * update * delete -* function (stored function) + You can read about the full documentation [here](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#yql-opentables-select) @@ -110,28 +110,70 @@ You can read about the full documentation [here](https://developer.yahoo.com/yql #### *Binder.addInput(input_object)* : Add an input object to the binder +```python +>>> song = InputKey(id='song', type='xs:string', paramType='path', required=True) +>>> select.addBinder(song) +``` #### *Binder.removeInput(input_id, input_type)* Remove an input object from the binder. ***input_type*** may be ***key, value or map*** +```python +>>> select.removeBinder('song','select') +``` #### *Binder.addUrl(url)* Add an url to the binder +```python +>>> select.addUrl('http://lol.com/{song}') +``` #### *Binder.removeUrl(url)* Remove an url from the binder +```python +>>> select.removeUrl('http://lol.com/{song}') +``` #### *Binder.addPaging(paging_instance)* Add a paging to the binder -#### *Binder.removePaging(paging_instance)* +```python +>>> mypage = PagingUrl("ysearchresponse.nextpage") +>>> select.addPaging(mypage) +``` +#### *Binder.removePaging()* Remove a paging from the binder +```python +>>> select.removePaging() +``` + +## BinderFunction + +This class represents a stored function. Read the full documentation [here](https://developer.yahoo.com/yql/guide/yql-opentables-reference.html#reference-function) + +### **Definition** + +#### *BinderFunction(func_name, func_code='', func_file=None, inputs=[])* + +* ***func_name*** : function name +* ***func_code*** : function code passed as string +* ***func_file*** : file containing the function +* ***inputs*** : list of inputs + +```python +>>> myfunc = BinderFunction('concat', func_code="console.log('Hello Josh!!!')") +``` + +### **Methods** + +As ***Binder***, ***BinderFunction*** is a subclass of ***BaseBinder***. They both share the same methods -## MetaClasses +## Using MetaClasses to define a Table + +***BinderModel*** and ***TableModel*** are the only classes to keep in mind here. They're respectively subclasses of ***BinderMeta*** and ***TableMeta***. Those last two help providing a powerful API to define YQL Table. -They say *"A picture is worth a thousand of word"* and I say *"A code snippet -is worth ..."* . You got it (^_^). -***BinderModel*** and ***TableModel*** are the only classes to use here. +They say *"A picture is worth a thousand of words"* and I say *"A code snippet +is worth ..."* . You got it (^_^). -Copy and past the code snippet below in a *example.py* +Copy and paste the code snippet below in a *example.py* ```python -from binder import BinderModel, InputKey, PagingPage, PagingUrl, InputValue, BinderFunction -from table import TableModel, BinderFrom +from myql.contrib.table.binder import BinderModel, InputKey, PagingPage, PagingUrl, InputValue, BinderFunction +from myql.contrib.table import TableModel, BinderFrom class SelectBinder(BinderModel): name = 'select' From 8962d1a7459ed116f080b79b7df192618274f444 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 1 May 2015 21:26:35 +0200 Subject: [PATCH 234/582] rauth installed --- requirements.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/requirements.txt b/requirements.txt index 235213f..a7543ef 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,17 @@ +Jinja2==2.7.3 +Markdown==2.6.2 +MarkupSafe==0.23 +PyYAML==3.11 +backports.ssl-match-hostname==3.4.0.2 certifi==14.05.14 +ghp-import==0.4.1 +livereload==2.3.2 +mkdocs==0.12.2 oauthlib==0.7.2 python-dateutil==2.2 +rauth==0.7.1 requests==2.4.0 requests-oauthlib==0.4.2 six==1.9.0 +tornado==4.1 wsgiref==0.1.2 From 2f5eeb2b7108195c75dd7abb0bf2effe2372a294 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 21:23:53 +0200 Subject: [PATCH 235/582] #63 : yaouth added --- myql/contrib/auth/yoauth.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 myql/contrib/auth/yoauth.py diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py new file mode 100644 index 0000000..d9e840a --- /dev/null +++ b/myql/contrib/auth/yoauth.py @@ -0,0 +1,31 @@ +""" +YOAuth is inspired from Darren Kempiners YahooAPI https://github.com/josuebrunel/python-yahooapi/blob/master/yahooapi.py +""" +import rauth +from rauth.utils import parse_utf8_qsl + +BASE_URL = "http://query.yahooapis.com/v1/yql" +REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token" +ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token" +AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth?oauth_token=" +CALLBACK_URI = 'oob' + + +class YOAuth(object): + """ + """ + + #def __init__(self, consumer_key=None, consumer_secret=None, access_token=None, access_token_secret=None, from_file=None): + def __init__(**kwargs): + """ + consumer_key : client key + consumer_secret : client secret + access_token : access token + access_token_secret : access token secret + from_file : file containing the credentials + """ + + pass + + + From d6ca5305e3b5e9613ba3c2246b187dd4abfcea2e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 22:35:16 +0200 Subject: [PATCH 236/582] fix #63 : OAuth 100% OK --- myql/contrib/auth/__init__.py | 1 + myql/contrib/auth/auth.py | 3 +- myql/contrib/auth/yoauth.py | 93 +++++++++++++++++++++++++++++++++-- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/myql/contrib/auth/__init__.py b/myql/contrib/auth/__init__.py index 05d034a..b653833 100755 --- a/myql/contrib/auth/__init__.py +++ b/myql/contrib/auth/__init__.py @@ -1 +1,2 @@ from auth import OAuth +from yoauth import YOAuth diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py index 52e9b47..af33348 100644 --- a/myql/contrib/auth/auth.py +++ b/myql/contrib/auth/auth.py @@ -74,7 +74,8 @@ def isValid(self): def refresh_token(self): """Refresh access token """ - oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token,resource_owner_secret=self.access_token_secret) + #oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token,resource_owner_secret=self.access_token_secret) + oauth = OAuth1(resource_owner_key=self.access_token,resource_owner_secret=self.access_token_secret) response = requests.post(ACCESS_TOKEN_URL, headers={'oauth_session_handle': self.session_handle}, auth=oauth) pdb.set_trace() tokens = self.fetch_tokens(response.content) diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index d9e840a..2e361b2 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -1,7 +1,12 @@ """ YOAuth is inspired from Darren Kempiners YahooAPI https://github.com/josuebrunel/python-yahooapi/blob/master/yahooapi.py """ -import rauth +import json +import time +import logging +import webbrowser + +from rauth import OAuth1Service from rauth.utils import parse_utf8_qsl BASE_URL = "http://query.yahooapis.com/v1/yql" @@ -10,13 +15,14 @@ AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth?oauth_token=" CALLBACK_URI = 'oob' +logging.basicConfig(level=logging.DEBUG, format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s") +logging.getLogger(__name__) + class YOAuth(object): """ """ - - #def __init__(self, consumer_key=None, consumer_secret=None, access_token=None, access_token_secret=None, from_file=None): - def __init__(**kwargs): + def __init__(self, consumer_key, consumer_secret, **kwargs): """ consumer_key : client key consumer_secret : client secret @@ -24,8 +30,85 @@ def __init__(**kwargs): access_token_secret : access token secret from_file : file containing the credentials """ + if kwargs.get('from_file'): + self.from_file = kwargs.get('from_file') + json_data = self.json_get_data(self.from_file) + vars(self).update(json_data) + else: + self.consumer_key = consumer_key + self.consumer_secret = consumer_secret + vars(self).update(kwargs) + + # Init OAuth + self.oauth = OAuth1Service( + consumer_key = self.consumer_key, + consumer_secret = self.consumer_secret, + name = "yahoo", + request_token_url = REQUEST_TOKEN_URL, + access_token_url = ACCESS_TOKEN_URL, + authorize_url = AUTHORIZE_TOKEN_URL, + base_url = BASE_URL + ) + + if vars(self).get('access_token') and vars(self).get('access_token_secret') and vars(self).get('session_handle'): + self.session = self.refresh_token() # Even though it hasn't expired + else: + # Fetching request token/token_secret + request_token, request_token_secret = self.oauth.get_request_token(params={'oauth_callback': CALLBACK_URI}) + logging.debug("REQUEST_TOKEN = {0}\n REQUEST_TOKEN_SECRET = {1}\n".format(request_token, request_token_secret)) + authorize_url = self.oauth.get_authorize_url(request_token) + logging.debug(authorize_url) + verifier = raw_input("Enter verifier : ") + logging.debug("VERIFIER = {0}".format(verifier)) + + self.token_time = time.time() + raw_acess = self.oauth.get_raw_access_token(request_token, request_token_secret, params={"oauth_verifier": verifier}) + parsed_acess = parse_utf8_qsl(raw_acess.content) + + self.access_token = parsed_acess['oauth_token'] + self.access_token_secret = parsed_acess['oauth_token_secret'] + self.session_handle = parsed_acess['oauth_session_handle'] + + json_data.update({ + 'access_token' : self.access_token, + 'access_token_secret' : self.access_token_secret, + 'session_handle' : self.session_handle, + 'token_time' : self.token_time + }) + + self.json_wirte_data(json_data, self.from_file) + + self.session = self.oauth.get_session((self.access_token, self.access_token_secret)) + + def json_get_data(self, filename): + """Returns content of a json file + """ + with open(filename) as fp: + json_data = json.load(fp) + + return json_data + + def json_wirte_data(self, json_data, filename): + """Write data into a json file + """ + with open(filename, 'w') as fp: + json.dump(json_data, fp, indent=4, encoding= 'utf-8', sort_keys=True) + return True + + return False + + def refresh_token(self,): + """Refresh access token + """ + logging.debug("REFRESHING TOKEN") + self.access_token_time = time.time() + self.acess_token, self.access_token_secret = self.oauth.get_access_token(self.access_token, self.access_token_secret, params={"oauth_session_handle": self.session_handle}) + + session = self.oauth.get_session((self.access_token, self.access_token_secret)) - pass + return session +if '__main__' == __name__: + auth = YOAuth(None, None, from_file='credentials.json') From 96a7383736a672a10e72cc51e0ea020e7b5f0f83 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 23:02:06 +0200 Subject: [PATCH 237/582] #63 : fix invalid_credential issue --- myql/contrib/auth/credentials.json | 8 ++++++++ myql/contrib/auth/yoauth.py | 17 +++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 myql/contrib/auth/credentials.json diff --git a/myql/contrib/auth/credentials.json b/myql/contrib/auth/credentials.json new file mode 100644 index 0000000..9291723 --- /dev/null +++ b/myql/contrib/auth/credentials.json @@ -0,0 +1,8 @@ +{ + "access_token": "A=2f2hZij6lwjdUdQRA40Wl31JLHBcl3hZBlT4W7LHsXblgaMN2qBkdDS.QFFzzhpdcA0aqaIvDgH0FpMytoiyJZ70JLnjMxGy3uzwonXqTXpBVrUSacCjdGHLv3p1SxBN6lnUtqfy5ZlTfNIbygGoTUW1XNzylj_KXN5.LhAHlz6P9PWA4paV6Fm4NgZCHd7_1KAirPvbhUJZOIs3HGkXZlADcpYaRHLPGzjO1Gzor5cVO97rg.r0u488Ha1n1EmTxGvL9KN4KPemxygEVxN.VyPL20.W9D9gIaJ2SMaPBaEHDICWgYj.SNCXm00eim6QLIFycjlMA84fgSjv7HlU0tmu_G6nMSLwTRO7Ogy0UsDMGNHP2Mz9KAzVp4zBHkYGeMygT1oeYZuRxkRLnQUsX0WtTLZ1n5G5tQSIVK9N6DoY0kgyuen.YoLtPz34bE8B6mXeuLeQ9lRfK62WNfkpddU8OHZ9NuHY662TRDglgKhp2yQL_gi2rQ7tXOPd6d0o2M7GfPcMDcwq47JIWBxpD.9kHUAI_Zbu4NGEPQ0JBUQLSAhdBRDB1g.KVkBdy98rgKTTJ4CgAJUbtLY_z1cM3dCBVKmId3.apoz34uYwJCGwN5JDOj5q6NWNjrd2M.S9ZkOyW82560cUaG6siNqAHGSch8km9ZtgEEYar_OUt.52.luW4jseqZVTcDiJs8XzdJ1fFarsJlyvuAt1eGoM1Gixq8yknFBmNJi4TpiKlXvJHFzfNuDg8_qB2R0aUYMCbXbRTYj4H1sPzHOQsdUQe2O1m0C35P2Sao9IRbdARoqrmA--", + "access_token_secret": "b0a0883ab686632588e9f9bf3ad837052218ff5e", + "consumer_key": "dj0yJmk9eFJINERDYWk2M3NkJmQ9WVdrOWEyNW1VRmRGTnpZbWNHbzlNQS0tJnM9Y29uc3VtZXJzZWNyZXQmeD1iNQ--", + "consumer_secret": "08802b459ab48eeaca765c119b0af6a4b75789f7", + "session_handle": "APIENFXij.bjFW1tEcr2YaGl91r3E_U_yOZDpZEXn.4.DOIYOR37", + "token_time": 1430598662.338279 +} \ No newline at end of file diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index 2e361b2..4772818 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -69,16 +69,17 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): self.access_token_secret = parsed_acess['oauth_token_secret'] self.session_handle = parsed_acess['oauth_session_handle'] - json_data.update({ - 'access_token' : self.access_token, - 'access_token_secret' : self.access_token_secret, - 'session_handle' : self.session_handle, - 'token_time' : self.token_time - }) + self.session = self.oauth.get_session((self.access_token, self.access_token_secret)) - self.json_wirte_data(json_data, self.from_file) + json_data.update({ + 'access_token' : self.access_token, + 'access_token_secret' : self.access_token_secret, + 'session_handle' : self.session_handle, + 'token_time' : self.token_time + }) + + self.json_wirte_data(json_data, self.from_file) - self.session = self.oauth.get_session((self.access_token, self.access_token_secret)) def json_get_data(self, filename): """Returns content of a json file From 0b5645825e155db5e2b1c1eedf39d2ef72125e8c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 23:05:44 +0200 Subject: [PATCH 238/582] cleaning the oauth house --- myql/contrib/auth/readme.md | 79 ------------------------------------- myql/myql.py | 6 +-- tests/tests.py | 7 ++-- 3 files changed, 6 insertions(+), 86 deletions(-) delete mode 100755 myql/contrib/auth/readme.md diff --git a/myql/contrib/auth/readme.md b/myql/contrib/auth/readme.md deleted file mode 100755 index 0e8ad1f..0000000 --- a/myql/contrib/auth/readme.md +++ /dev/null @@ -1,79 +0,0 @@ -OAuth -========== - -***Auth*** is 3 legged *OAuth* module dedicated to the *Yahoo! OAuth Provider*. It uses [rauth](http://rauth.readthedocs.org/en/latest/) as *oauth* library. - -You only need **2** parameters to make it work : -- Your ***consumer key*** -- Your ***consumer secret*** - -I tried to make it as painless as possible for me (and you too ;) ). Thus **3 legged OAuth** , **3 methods** and **3 steps** ( in order ) : -* Auth.get_request_token() -* Auth.get_user_authorization() -* Auth.get_access_token() - -Quick start ------------ - -```python ->>> from myql import OAuth ->>> from myql.config import consumer_key, consumer_secret # config file where you have your consumer_key and consumer_secret ->>> consumer_key -'dj0yJmk9aVRSd3ZabElmTzJNJmQ9WVdrOWEyNW1VRmRGTnpZbWNHbzlNQS0tJnM9Y29uc3VtZXJzZWNyZXQmeD1hMg--' ->>> consumer_secret -'***************************' # Sorry guys it's kinda personal ->>> auth = Auth(consumer_key, consumer_secret) ->>> #Step 1:Getting request token -... ->>> auth.get_request_token() -(u'hjmbdag', u'cdb5688ed78a54e4e87f8456580f8197b2069270') ->>> #Step 2: Getting User Authorization -... ->>> auth.get_user_authorization() -``` - -* Granting access - -![Alt grant](../../../static/img/grant.png) - -* Getting code - -![Alt code](../../../static/img/code.png) - -```python -Please input the verifier : 2k8akm -'2k8akm' ->>> #Step 3:Getting access token -... ->>> auth.get_access_token() -(u'A=pxautZv7kC.gB_I7UvbIv9GN1XB2Ky7Ql1EOie0KTzvjKTZfd5OpqlKF3WLRhEz1qxyUfbtQsCELg5NqrHHcnA8n8CytqMfZsTLLJLFi4mKZX6L3R4xbt1jbTY_dW6wM5ffz18SJOHLQVhFXjC2U826fndT.eBpVh5hJ8QGcxKYaEiqhwue7LqXtFEUnSEzDsCebR5ZsUsi_T7dKHZ9DL4tNfpi81Il0o9xkakkJt9i2raXrC49J7Ds8tUpjkhSbuay_HcDLeZwXOW4WN1TXIP_6ZofAP9dzdD3mOm.u9wONzontMraUjE6wSic8k0UfOHvcIJTV5JITpYjjw7BX3r.NU119rZqo_VGpIDecZmPXkRKy6w.4g9xfBize0hgh8118j5qXMbSW50bOwhTDyF2k3wVfhBc9qYwUTgsFE1GJeJCx0jU2Y6re6OuOp4NDmTZsaCm1pG7D180nvGq_5j3Tf0OSYRo6noffhbMZ.KnnkBRdu9a0.a5GCBC4RKKBUtd4EW7zNF5sODlVLjisa4RZ5XwzfKSafmNrAeSibVc.WRDhleMziKcf3jPmafHx09xbCfDWUg8FOMOKWaJzr_ocjUIqPXQUG6ryzRw61IkajCvQ_LGLa_q3eBwT3WCxTOBm2x9hb6Hw1CVTvV_CbeevE7jGrcyJH6UH69YgdpG0A1vGFhLSRh.bpiyTuxYCxFByZKxR8onSblcY6wG5NDq_kbcmtyYFmIVLoPqPk9SCuBeRQYpMoyumv0U8FfJVvijE8b41PCzEKexCrfLdmbKYDmsFaeS_oghC0WjkGCZVQX7nmt.XJYntr8dzdVBqJcU6YG0NGpYGQp_r9B_0vlqmjfj61fXkFIxGsNTMArOShw--', u'795a47a1e1dc2385130ec1e738177dd9694d970a') ->>> -``` - -As you noticed, all we needed were a *consumer_key* and *consumer_secret* . - -Access to some attributes (it's just python) ---------------------------------------------- - -```python ->>> auth.request_token -u'hjmbdag' ->>> auth.request_token_secret -u'cdb5688ed78a54e4e87f8456580f8197b2069270' ->>> auth.verifier -'2k8akm' ->>> auth.access_token -u'A=pxautZv7kC.gB_I7UvbIv9GN1XB2Ky7Ql1EOie0KTzvjKTZfd5OpqlKF3WLRhEz1qxyUfbtQsCELg5NqrHHcnA8n8CytqMfZsTLLJLFi4mKZX6L3R4xbt1jbTY_dW6wM5ffz18SJOHLQVhFXjC2U826fndT.eBpVh5hJ8QGcxKYaEiqhwue7LqXtFEUnSEzDsCebR5ZsUsi_T7dKHZ9DL4tNfpi81Il0o9xkakkJt9i2raXrC49J7Ds8tUpjkhSbuay_HcDLeZwXOW4WN1TXIP_6ZofAP9dzdD3mOm.u9wONzontMraUjE6wSic8k0UfOHvcIJTV5JITpYjjw7BX3r.NU119rZqo_VGpIDecZmPXkRKy6w.4g9xfBize0hgh8118j5qXMbSW50bOwhTDyF2k3wVfhBc9qYwUTgsFE1GJeJCx0jU2Y6re6OuOp4NDmTZsaCm1pG7D180nvGq_5j3Tf0OSYRo6noffhbMZ.KnnkBRdu9a0.a5GCBC4RKKBUtd4EW7zNF5sODlVLjisa4RZ5XwzfKSafmNrAeSibVc.WRDhleMziKcf3jPmafHx09xbCfDWUg8FOMOKWaJzr_ocjUIqPXQUG6ryzRw61IkajCvQ_LGLa_q3eBwT3WCxTOBm2x9hb6Hw1CVTvV_CbeevE7jGrcyJH6UH69YgdpG0A1vGFhLSRh.bpiyTuxYCxFByZKxR8onSblcY6wG5NDq_kbcmtyYFmIVLoPqPk9SCuBeRQYpMoyumv0U8FfJVvijE8b41PCzEKexCrfLdmbKYDmsFaeS_oghC0WjkGCZVQX7nmt.XJYntr8dzdVBqJcU6YG0NGpYGQp_r9B_0vlqmjfj61fXkFIxGsNTMArOShw--' ->>> auth.access_token_secret -u'795a47a1e1dc2385130ec1e738177dd9694d970a' ->>> -``` - -2 Legged OAuth (Coming pretty soon) ------------------------------------ - -In *2 legged OAuth* the **User Authorization** isn't required anymore ... - -Voila ... - - diff --git a/myql/myql.py b/myql/myql.py index 3b720ac..51c3192 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -1,5 +1,5 @@ import requests -from contrib.auth import OAuth +from contrib.auth import YOAuth import errors import importlib @@ -84,9 +84,7 @@ def executeQuery(self, payload): '''Execute the query and returns and response''' if vars(self).get('oauth'): self.url = self.private_url - if not self.oauth.isValid(): - print("Reset your credentials : they don't seem to be valid anymore") - response = requests.get(self.url, params= payload, auth= self.oauth.oauth) + response = self.oauth.session.get(self.url, params= payload) else: response = requests.get(self.url, params= payload) diff --git a/tests/tests.py b/tests/tests.py index 05071e6..5c8a90c 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -5,7 +5,7 @@ from xml.etree import cElementTree as xtree from myql import MYQL -from myql.contrib.auth import OAuth +from myql.contrib.auth import YOAuth from myql.contrib.table import Table from myql.contrib.table import Base, BaseInput @@ -27,10 +27,11 @@ def tearUp(self,): pass def test_get_guid(self,): - oauth = OAuth(None, None, from_file='credentials.json') + oauth = YOAuth(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) response = yql.getGUID('josue_brunel') - logging.debug(response.status_code,200) + logging.debug(response.content) + self.assertEquals(response.status_code,200) class TestTable(unittest.TestCase): From 4d1c6efad280ad65fa5ec0dfa26369d39a91bc47 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 23:23:19 +0200 Subject: [PATCH 239/582] #63 : fix checking to token validity --- myql/contrib/auth/yoauth.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index 4772818..139e328 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -51,7 +51,10 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): ) if vars(self).get('access_token') and vars(self).get('access_token_secret') and vars(self).get('session_handle'): - self.session = self.refresh_token() # Even though it hasn't expired + if not self.token_is_valid(): + self.session = self.refresh_token() + else: + self.session = self.oauth.get_session((self.access_token, self.access_token_secret)) else: # Fetching request token/token_secret request_token, request_token_secret = self.oauth.get_request_token(params={'oauth_callback': CALLBACK_URI}) @@ -109,6 +112,18 @@ def refresh_token(self,): return session + def token_is_valid(self,): + """Check the validity of the token :3600s + """ + elapsed_time = time.time() - self.token_time + if elapsed_time > 3600: + logging.debug("TOKEN HAS EXPIRED") + return False + + logging.debug("TOKEN IS STILL VALID") + return True + + if '__main__' == __name__: auth = YOAuth(None, None, from_file='credentials.json') From d3d89314c81c0ad01a10c861391025c3d818b3e8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 23:25:09 +0200 Subject: [PATCH 240/582] #12 : refreshing token while querying --- myql/myql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/myql/myql.py b/myql/myql.py index 51c3192..960ca7c 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -84,6 +84,8 @@ def executeQuery(self, payload): '''Execute the query and returns and response''' if vars(self).get('oauth'): self.url = self.private_url + if not self.oauth.token_is_valid(): # Refresh token if token has expired + self.oauth.refresh_token() response = self.oauth.session.get(self.url, params= payload) else: response = requests.get(self.url, params= payload) From 936675358561bb5843bdc29afe08253e5d336156 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 23:36:57 +0200 Subject: [PATCH 241/582] #12 : fix update of token_time --- myql/contrib/auth/yoauth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index 139e328..b44ff9d 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -105,7 +105,7 @@ def refresh_token(self,): """Refresh access token """ logging.debug("REFRESHING TOKEN") - self.access_token_time = time.time() + self.token_time = time.time() self.acess_token, self.access_token_secret = self.oauth.get_access_token(self.access_token, self.access_token_secret, params={"oauth_session_handle": self.session_handle}) session = self.oauth.get_session((self.access_token, self.access_token_secret)) @@ -116,6 +116,7 @@ def token_is_valid(self,): """Check the validity of the token :3600s """ elapsed_time = time.time() - self.token_time + logging.debug("ELAPSED TIME : {0}".format(elapsed_time)) if elapsed_time > 3600: logging.debug("TOKEN HAS EXPIRED") return False From 9a5ec3ad07c70c95ad8b2bc22f3b6ca564c9e301 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 23:45:34 +0200 Subject: [PATCH 242/582] cleaning stuff --- myql/contrib/auth/credentials.json | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 myql/contrib/auth/credentials.json diff --git a/myql/contrib/auth/credentials.json b/myql/contrib/auth/credentials.json deleted file mode 100644 index 9291723..0000000 --- a/myql/contrib/auth/credentials.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "access_token": "A=2f2hZij6lwjdUdQRA40Wl31JLHBcl3hZBlT4W7LHsXblgaMN2qBkdDS.QFFzzhpdcA0aqaIvDgH0FpMytoiyJZ70JLnjMxGy3uzwonXqTXpBVrUSacCjdGHLv3p1SxBN6lnUtqfy5ZlTfNIbygGoTUW1XNzylj_KXN5.LhAHlz6P9PWA4paV6Fm4NgZCHd7_1KAirPvbhUJZOIs3HGkXZlADcpYaRHLPGzjO1Gzor5cVO97rg.r0u488Ha1n1EmTxGvL9KN4KPemxygEVxN.VyPL20.W9D9gIaJ2SMaPBaEHDICWgYj.SNCXm00eim6QLIFycjlMA84fgSjv7HlU0tmu_G6nMSLwTRO7Ogy0UsDMGNHP2Mz9KAzVp4zBHkYGeMygT1oeYZuRxkRLnQUsX0WtTLZ1n5G5tQSIVK9N6DoY0kgyuen.YoLtPz34bE8B6mXeuLeQ9lRfK62WNfkpddU8OHZ9NuHY662TRDglgKhp2yQL_gi2rQ7tXOPd6d0o2M7GfPcMDcwq47JIWBxpD.9kHUAI_Zbu4NGEPQ0JBUQLSAhdBRDB1g.KVkBdy98rgKTTJ4CgAJUbtLY_z1cM3dCBVKmId3.apoz34uYwJCGwN5JDOj5q6NWNjrd2M.S9ZkOyW82560cUaG6siNqAHGSch8km9ZtgEEYar_OUt.52.luW4jseqZVTcDiJs8XzdJ1fFarsJlyvuAt1eGoM1Gixq8yknFBmNJi4TpiKlXvJHFzfNuDg8_qB2R0aUYMCbXbRTYj4H1sPzHOQsdUQe2O1m0C35P2Sao9IRbdARoqrmA--", - "access_token_secret": "b0a0883ab686632588e9f9bf3ad837052218ff5e", - "consumer_key": "dj0yJmk9eFJINERDYWk2M3NkJmQ9WVdrOWEyNW1VRmRGTnpZbWNHbzlNQS0tJnM9Y29uc3VtZXJzZWNyZXQmeD1iNQ--", - "consumer_secret": "08802b459ab48eeaca765c119b0af6a4b75789f7", - "session_handle": "APIENFXij.bjFW1tEcr2YaGl91r3E_U_yOZDpZEXn.4.DOIYOR37", - "token_time": 1430598662.338279 -} \ No newline at end of file From f47f340a45e810ed370597a39eed54ca62f90e37 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 23:48:55 +0200 Subject: [PATCH 243/582] fixing thank you link --- myql/contrib/auth/yoauth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index b44ff9d..bde544a 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -1,5 +1,5 @@ """ -YOAuth is inspired from Darren Kempiners YahooAPI https://github.com/josuebrunel/python-yahooapi/blob/master/yahooapi.py +YOAuth is inspired from Darren Kempiners YahooAPI https://github.com/dkempiners/python-yahooapi/blob/master/yahooapi.py """ import json import time From 488e7bd53d3d60f05bd18dddde7bbbf0e9a14650 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 2 May 2015 23:54:40 +0200 Subject: [PATCH 244/582] #63 : webbrowser action added --- myql/contrib/auth/yoauth.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index bde544a..638933c 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -59,8 +59,10 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): # Fetching request token/token_secret request_token, request_token_secret = self.oauth.get_request_token(params={'oauth_callback': CALLBACK_URI}) logging.debug("REQUEST_TOKEN = {0}\n REQUEST_TOKEN_SECRET = {1}\n".format(request_token, request_token_secret)) - authorize_url = self.oauth.get_authorize_url(request_token) + #authorize_url = self.oauth.get_authorize_url(request_token) + authorize_url = AUTHORIZE_TOKEN_URL+request_token logging.debug(authorize_url) + webbrowser.open(authorize_url) verifier = raw_input("Enter verifier : ") logging.debug("VERIFIER = {0}".format(verifier)) From 8788b69ff2d07165bb56f9462b8da46f673b17a4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 01:02:20 +0200 Subject: [PATCH 245/582] updating the documentation --- README.md | 13 ++++++++++++- docs/index.md | 10 ++++++++++ docs/myql.md | 2 ++ docs/oauth.md | 29 +++++++++++++++++++++++++++++ docs/table.md | 2 +- mkdocs.yml | 4 +++- 6 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 docs/myql.md create mode 100644 docs/oauth.md diff --git a/README.md b/README.md index 3902f80..85eca8f 100755 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ installation ============ ```shell -$ python setup.py install +$ pip install myql ``` how to use @@ -153,3 +153,14 @@ Same as ***SELECT***, but instead returns data. >>> ``` +Using OAuth to fetch protected resources +========================================= + +```python +>>> from myql.contrib.auth import YOAuth +>>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. +>>> from myql import MYQL +>>> yql = MYQL(format='xml', oauth=oauth) +>>> response = yql.getGUID('josue_brunel') # Deal with the response +``` + diff --git a/docs/index.md b/docs/index.md index 4c7f589..c60253e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -137,3 +137,13 @@ Same as ***SELECT***, but instead returns data. >>> ``` +Using OAuth to fetch protected resources +========================================= + +```python +>>> from myql.contrib.auth import YOAuth +>>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. +>>> from myql import MYQL +>>> yql = MYQL(format='xml', oauth=oauth) +>>> response = yql.getGUID('josue_brunel') # Deal with the response +``` diff --git a/docs/myql.md b/docs/myql.md new file mode 100644 index 0000000..3f3e39c --- /dev/null +++ b/docs/myql.md @@ -0,0 +1,2 @@ +MYQL +==== \ No newline at end of file diff --git a/docs/oauth.md b/docs/oauth.md new file mode 100644 index 0000000..54e0162 --- /dev/null +++ b/docs/oauth.md @@ -0,0 +1,29 @@ +YOAuth as Yahoo OAuth +===================== + +Before going any further i would like to thank [Darren Kempiners](https://github.com/dkempiners) and [Andrew Martin](https://github.com/almartin82) for their help. + +## YOAuth + +This class is used to generates an oauth *session* which will be used in your requests to the *YQL Service* + +### **Definition** + +#### *YOAuth(consumer_key, consumer_secret, \*\*kwargs)* + +* ***consumer_key*** : Client Key of your application. +* ***consumer_secret*** : Client Secret of your application +* ***acess_token*** : The Access Token +* ***acess_token_secret*** : The Acess Token Secret +* ***session_handler*** : The OAuth Session Handler which is required when refreshing the token +* ***from_file*** : File containing the credentials. + +The minimum information required in a ***credentials file*** are the ***consumer_key*** and the ***consumer_secret***. +If ***from_file*** is provided, the class will be instanciated with data within this file. + +### **Methods** + +- #### *OAuth.json_get_data(filename)* +- #### *OAuth.json_write_data(json_data, filename)* +- #### *OAuth.token_is_valid()* +- #### *OAuth.refresh_token()* diff --git a/docs/table.md b/docs/table.md index 43cd65f..b62443d 100644 --- a/docs/table.md +++ b/docs/table.md @@ -172,7 +172,7 @@ is worth ..."* . You got it (^_^). Copy and paste the code snippet below in a *example.py* ```python -from myql.contrib.table.binder import BinderModel, InputKey, PagingPage, PagingUrl, InputValue, BinderFunction +from myql.contrib.table import BinderModel, InputKey, PagingPage, PagingUrl, InputValue, BinderFunction from myql.contrib.table import TableModel, BinderFrom class SelectBinder(BinderModel): diff --git a/mkdocs.yml b/mkdocs.yml index 43f7214..88fb5b9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,8 @@ site_name: MYQL pages: - [index.md, Home] -- [table.md, OpenTable] +- [myql.md, MYQL] +- [oauth.md, YOAuth] +- [table.md, Open Table] - [contrib.md, Contribute] theme: readthedocs From 7dd4c91d3e74ffaf49d8c8fd6100762b0b900efc Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 06:00:05 +0200 Subject: [PATCH 246/582] updating gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a2063ee..49b33e8 100755 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,4 @@ docs/_build/ myconfig.py *.xml site +*.json From a4db10f4d0329275ee2e2afc858969f15b4605f1 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 07:20:48 +0200 Subject: [PATCH 247/582] #1: updating mkdocs config --- mkdocs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 88fb5b9..2c5b4f5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,10 @@ site_name: MYQL +repo_url: https://github.com/josuebrunel/myql +repo_name: https://github.com/josuebrunel/myql +site_url: http://myql.readthedocs.org/en/latest/myql/ +site_description: MYQL documentation +site_author: Josue Kouka +google_analytics: ['UA-32441224-4', 'readthedocs.org'] pages: - [index.md, Home] - [myql.md, MYQL] From a3edbf9ed2f92ac75e354eb35bcc0800882c6b7e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 13:08:01 +0200 Subject: [PATCH 248/582] #1 : updating doc --- README.md | 4 +-- docs/index.md | 10 +++--- docs/myql.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++- docs/oauth.md | 23 +++++++++++-- mkdocs.yml | 1 - 5 files changed, 118 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 85eca8f..0c5753d 100755 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ MYQL is a Python wrapper of the Yahoo Query Language. Yahoo! Query Language Documentation and Support =============================================== -* Yahoo! Query Language - http://developer.yahoo.com/yql/ -* Yahoo! Developer Network: http://developer.yahoo.com +* Yahoo! Query Language - (http://developer.yahoo.com/yql/) +* Yahoo! Developer Network: (http://developer.yahoo.com) * Yahoo! Application Platform - http://developer.yahoo.com/yap/ * Yahoo! Social APIs - http://developer.yahoo.com/social/ * Yahoo! QUery Language Console https://developer.yahoo.com/yql/console/ diff --git a/docs/index.md b/docs/index.md index c60253e..eb8c93b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,11 +6,11 @@ MYQL is a Python wrapper of the Yahoo Query Language. Yahoo! Query Language Documentation and Support =============================================== -* Yahoo! Query Language - http://developer.yahoo.com/yql/ -* Yahoo! Developer Network: http://developer.yahoo.com -* Yahoo! Application Platform - http://developer.yahoo.com/yap/ -* Yahoo! Social APIs - http://developer.yahoo.com/social/ -* Yahoo! Query Language Console https://developer.yahoo.com/yql/console/ +* [Yahoo! Query Language](http://developer.yahoo.com/yql/) +* [Yahoo! Developer Network](http://developer.yahoo.com) +* [Yahoo! Application Platform](http://developer.yahoo.com/yap/) +* [Yahoo! Social APIs](http://developer.yahoo.com/social/) +* [Yahoo! Query Language Console](https://developer.yahoo.com/yql/console/) Installation ============ diff --git a/docs/myql.md b/docs/myql.md index 3f3e39c..3c82400 100644 --- a/docs/myql.md +++ b/docs/myql.md @@ -1,2 +1,92 @@ MYQL -==== \ No newline at end of file +==== + +### **Definition** + +#### *MYQL(community=True, fromat='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None)* + +### **Methods** + +#### *MYQL.payloadBuilder(query, format='json')* + +Return a dictionary of parameters + +* ***query*** : the YQL Query +* ***format*** : xml or json + +#### *MQYL.executeQuery(payload)* + +Execute the query and returns and response + +* ***payload*** : Dict of parameters + + +#### *MYQL.rawQuery(query, format=None, pretty=False)* + +Call *payloadBuilder* to build paramaters and *executeQuery* to execute que *query* then return a response. + +* ***query*** : the YQL Query +* ***format*** : xml or json + +#### *MYQL.clauseFormatter(condition)* + +Formats conditions. + +* ***condition*** : list of ['column', 'operator', 'value'] +```python +cond = ['yid', '=', 'josue_brunel'] +``` + + +#### *MQYL.buildResponse(response)* + +#### *MQYL.use(url)* + + Change the service provider + +* ***url*** : url of the service provider + + +#### *MQYL.desc(table=None)* + +Get the description of a table. +If no table name is provided, the **self.table** will be used. + +* ***table*** : Table name + +#### *MQYL.get(table=None, items=[], limit=None)* + +Get **items** from **table**. + +* ***table*** : Table name +* ***items*** : Element/columns to get from the table +* ***limit*** : limit of element to fetch + + +#### *MQYL.select(table=None, items=[], limit=None)* + +* ***table*** : Table name +* ***items*** : Element/columns to get from the table +* ***limit*** : limit of element to fetch + +```python +>>> yql.select('social.profile', ['guid', 'givenName', 'gender']) +``` + +#### *MQYL.where(\*args)* + +* ***\*args*** : List of conditions + +```python +>>> yql.select('mytable.friends').where(['name', '=', 'alain'], ['location', '!=', 'paris']) +``` + +#### *MQYL.showTables()* + +List all tables + +#### *MQYL.getGUID(username)* + +Return a user *guid* + +* ***username*** : yahoo id i.e 'josue_brunel' \ No newline at end of file diff --git a/docs/oauth.md b/docs/oauth.md index 54e0162..7d32c5c 100644 --- a/docs/oauth.md +++ b/docs/oauth.md @@ -3,9 +3,7 @@ YOAuth as Yahoo OAuth Before going any further i would like to thank [Darren Kempiners](https://github.com/dkempiners) and [Andrew Martin](https://github.com/almartin82) for their help. -## YOAuth - -This class is used to generates an oauth *session* which will be used in your requests to the *YQL Service* +The ***YOAuth*** class is used to generates an oauth *session* which will be used in your requests to the *YQL Service* ### **Definition** @@ -21,6 +19,25 @@ This class is used to generates an oauth *session* which will be used in your re The minimum information required in a ***credentials file*** are the ***consumer_key*** and the ***consumer_secret***. If ***from_file*** is provided, the class will be instanciated with data within this file. +```python +>>> from myql.contrib.auth import YOAuth +>>> oauth = YOAuth('khdhkfhbb7rit93ffhbfh', 'urysfjue76885hgf') +``` + +Using a credentials json file + +*credentials.json* +```json +{ + "consumer_key" : "khdhkfhbb7rit93ffhbfh", + "consumer_secret": "urysfjue76885hgf" +} +``` + +```python +>>> oauth = YOAuth(None, None, from_file=os.path.realpath('credentials.json')) +``` + ### **Methods** - #### *OAuth.json_get_data(filename)* diff --git a/mkdocs.yml b/mkdocs.yml index 2c5b4f5..e073cae 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,5 @@ site_name: MYQL repo_url: https://github.com/josuebrunel/myql -repo_name: https://github.com/josuebrunel/myql site_url: http://myql.readthedocs.org/en/latest/myql/ site_description: MYQL documentation site_author: Josue Kouka From 8630171f2469b10632ae6ea59713731dc2cf3a47 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 13:30:53 +0200 Subject: [PATCH 249/582] #1 : updating oauth doc --- docs/oauth.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/oauth.md b/docs/oauth.md index 7d32c5c..e5f1afe 100644 --- a/docs/oauth.md +++ b/docs/oauth.md @@ -16,7 +16,7 @@ The ***YOAuth*** class is used to generates an oauth *session* which will be use * ***session_handler*** : The OAuth Session Handler which is required when refreshing the token * ***from_file*** : File containing the credentials. -The minimum information required in a ***credentials file*** are the ***consumer_key*** and the ***consumer_secret***. +The **minimum** information required in a ***credentials file*** are the ***consumer_key*** and the ***consumer_secret***. If ***from_file*** is provided, the class will be instanciated with data within this file. ```python @@ -41,6 +41,17 @@ Using a credentials json file ### **Methods** - #### *OAuth.json_get_data(filename)* + +Return a dict containing the credentials + - #### *OAuth.json_write_data(json_data, filename)* + +Update the credentials file + - #### *OAuth.token_is_valid()* + +Check if the token is still valid + - #### *OAuth.refresh_token()* + +Refresh the expired token \ No newline at end of file From b8e8e41696fb52ae5ac84e7be323a6c0e6f23ef6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 13:31:19 +0200 Subject: [PATCH 250/582] updating setup config --- setup.cfg | 2 ++ setup.py | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..aac93ff --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] + description-file = README.md diff --git a/setup.py b/setup.py index a3931c5..58eadad 100755 --- a/setup.py +++ b/setup.py @@ -12,6 +12,8 @@ author = "Josue Kouka", author_email = "josuebrunel@gmail.com", url = "https://github.com/josuebrunel/MYQL", + download_url = "https://github.com/josuebrunel/myql/tarball/1.2", + keywords = ['myql', 'yql', 'yahoo', 'query', 'language'], packages = find_packages(), platforms=['Any'], license='BSD', From 7320cbf18d867fd4c3f5fd0beaa469f6bbee956b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 13:51:18 +0200 Subject: [PATCH 251/582] classifiers added --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 58eadad..f104898 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,13 @@ download_url = "https://github.com/josuebrunel/myql/tarball/1.2", keywords = ['myql', 'yql', 'yahoo', 'query', 'language'], packages = find_packages(), + classifiers = [ + 'Programming Language :: Python :: 2.7', + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License' + ], platforms=['Any'], license='BSD', install_requires = required From dc1dea1665f5ebca2090d4298c60d6c7ee2ba65d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 14:03:30 +0200 Subject: [PATCH 252/582] fixing config parser error --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index aac93ff..b88034e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [metadata] - description-file = README.md +description-file = README.md From 0471d3604d4f3a2c066d5d2552685f3fe06174c0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 14:11:16 +0200 Subject: [PATCH 253/582] reqierements updated --- requirements.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index a7543ef..235213f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,7 @@ -Jinja2==2.7.3 -Markdown==2.6.2 -MarkupSafe==0.23 -PyYAML==3.11 -backports.ssl-match-hostname==3.4.0.2 certifi==14.05.14 -ghp-import==0.4.1 -livereload==2.3.2 -mkdocs==0.12.2 oauthlib==0.7.2 python-dateutil==2.2 -rauth==0.7.1 requests==2.4.0 requests-oauthlib==0.4.2 six==1.9.0 -tornado==4.1 wsgiref==0.1.2 From d7834c83c461fa87a03bef302d12fa33f4f20cdd Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 14:15:28 +0200 Subject: [PATCH 254/582] fixing issue with rauth --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 235213f..436a279 100755 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ requests==2.4.0 requests-oauthlib==0.4.2 six==1.9.0 wsgiref==0.1.2 +rauth==0.7.1 From b24146026b3fce5495d709a337583c98e770531e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 14:28:09 +0200 Subject: [PATCH 255/582] changing download url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f104898..ab4d248 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ author = "Josue Kouka", author_email = "josuebrunel@gmail.com", url = "https://github.com/josuebrunel/MYQL", - download_url = "https://github.com/josuebrunel/myql/tarball/1.2", + download_url = "https://github.com/josuebrunel/myql/archives/1.2.tar.gz", keywords = ['myql', 'yql', 'yahoo', 'query', 'language'], packages = find_packages(), classifiers = [ From 48ae8d0f80169b43466163a645435f2b5e794548 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 14:58:48 +0200 Subject: [PATCH 256/582] updating requirements and script to publish --- publish_package.sh | 20 ++++++++++++++++++++ requirements.txt | 7 ++----- 2 files changed, 22 insertions(+), 5 deletions(-) create mode 100755 publish_package.sh diff --git a/publish_package.sh b/publish_package.sh new file mode 100755 index 0000000..b2d4be2 --- /dev/null +++ b/publish_package.sh @@ -0,0 +1,20 @@ +################################################## +# +# Author : josuebrunel +# Filename : publish_package.sh +# Description : +# Creation Date : 03-05-2015 +# Last Modified : Sun May 3 14:55:43 2015 +# +################################################## + +if [ ! -z $1 ]; then + if [ $1 == 'testing' ] + $server=pypitest + else + $server=pypi +fi + +python setup.py register -r $sever + +python setup.py sdist upload -r $server diff --git a/requirements.txt b/requirements.txt index 436a279..83a959f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ -certifi==14.05.14 oauthlib==0.7.2 -python-dateutil==2.2 -requests==2.4.0 +rauth==0.7.1 +requests==2.6.2 requests-oauthlib==0.4.2 -six==1.9.0 wsgiref==0.1.2 -rauth==0.7.1 From 23a6e246f24072339ea0f43d3dce6121e921d345 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 15:07:16 +0200 Subject: [PATCH 257/582] logging added --- publish_package.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/publish_package.sh b/publish_package.sh index b2d4be2..daeeaeb 100755 --- a/publish_package.sh +++ b/publish_package.sh @@ -4,17 +4,24 @@ # Filename : publish_package.sh # Description : # Creation Date : 03-05-2015 -# Last Modified : Sun May 3 14:55:43 2015 +# Last Modified : Sun May 3 15:07:04 2015 # ################################################## if [ ! -z $1 ]; then - if [ $1 == 'testing' ] - $server=pypitest + if [ "$1" == "testing" ]; then + server=pypitest else - $server=pypi + server=pypi + fi +else + server=pypitest fi -python setup.py register -r $sever +_debug "Publish on ${server}" +_info "python setup.py register -r ${server}" +python setup.py register -r $server + +_info "python setup.py sdist upload -r ${server}" python setup.py sdist upload -r $server From b01c8ea8ca25ac5dc6ac23f55f9498012ec2db08 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 17:03:46 +0200 Subject: [PATCH 258/582] PyPi badge added --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0c5753d..e3f6ddb 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ MYQL ========= -[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) +[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) [![PyPI version](https://badge.fury.io/py/myql.svg)](http://badge.fury.io/py/myql) MYQL is a Python wrapper of the Yahoo Query Language. @@ -9,8 +9,8 @@ MYQL is a Python wrapper of the Yahoo Query Language. Yahoo! Query Language Documentation and Support =============================================== -* Yahoo! Query Language - (http://developer.yahoo.com/yql/) -* Yahoo! Developer Network: (http://developer.yahoo.com) +* Yahoo! Query Language - http://developer.yahoo.com/yql/ +* Yahoo! Developer Network: http://developer.yahoo.com * Yahoo! Application Platform - http://developer.yahoo.com/yap/ * Yahoo! Social APIs - http://developer.yahoo.com/social/ * Yahoo! QUery Language Console https://developer.yahoo.com/yql/console/ From c4cbf8c8dede42e0a8eefa99d0a5985967824802 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 3 May 2015 17:04:59 +0200 Subject: [PATCH 259/582] using rst file for package description --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ab4d248..e83ffb6 100755 --- a/setup.py +++ b/setup.py @@ -1,14 +1,18 @@ +import os from setuptools import setup, find_packages #requirements.txt with open('requirements.txt') as f: required = f.read().splitlines() +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + setup( name = "myql", version = "1.2", description = "Python Wrapper for the Yahoo ! Query Language", - long_description = "", + long_description = read("README.rst"), author = "Josue Kouka", author_email = "josuebrunel@gmail.com", url = "https://github.com/josuebrunel/MYQL", From bbe2ca1caf947ba05b543ab5ffbce92849fb6f8e Mon Sep 17 00:00:00 2001 From: The Gitter Badger Date: Tue, 5 May 2015 07:47:18 +0000 Subject: [PATCH 260/582] Added Gitter badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e3f6ddb..d668328 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ MYQL ========= +[![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + [![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) [![PyPI version](https://badge.fury.io/py/myql.svg)](http://badge.fury.io/py/myql) From 6098fe5b2017b8bc5287ac1292e09558c8948f39 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 5 May 2015 10:35:50 +0200 Subject: [PATCH 261/582] gitter badge added --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index d668328..4904850 100755 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ MYQL ========= +[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) [![PyPI version](https://badge.fury.io/py/myql.svg)](http://badge.fury.io/py/myql) [![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) [![PyPI version](https://badge.fury.io/py/myql.svg)](http://badge.fury.io/py/myql) - MYQL is a Python wrapper of the Yahoo Query Language. From c292b1b2ed1a484e803e438321cebec6a12280ec Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 5 May 2015 17:20:29 +0200 Subject: [PATCH 262/582] Quantified code badge adde --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4904850..e162e6f 100755 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ MYQL ========= [![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) [![PyPI version](https://badge.fury.io/py/myql.svg)](http://badge.fury.io/py/myql) -[![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Issues](https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg)](https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql) MYQL is a Python wrapper of the Yahoo Query Language. From 86e0189f9352cb383056175d78d8149e61bf3e0d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 6 May 2015 08:20:14 +0200 Subject: [PATCH 263/582] test for yahoo fantasy sportadded --- tests/tests.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 5c8a90c..a313178 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,4 +1,4 @@ -import os, logging +import os, logging, time import pdb import unittest from xml.dom import minidom @@ -17,7 +17,6 @@ logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logging.getLogger(__name__) - class TestOAuth(unittest.TestCase): def setUp(self,): @@ -33,6 +32,16 @@ def test_get_guid(self,): logging.debug(response.content) self.assertEquals(response.status_code,200) + def test_yahoo_fantasy_sport(self,): + oauth = YOAuth(None, None, from_file='credentials.json') + yql = MYQL(format='json', oauth=oauth) + #response = yql.select('fantasysports.teams.roster').where(['team_key', '=', 'mlb.l.1328.t.1'], ['date', '=', '2015-05-05']) + response = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key IN ('mlb.l.1328.t.1','mlb.l.1328.t.2') AND date = '2015-05-05'") + pdb.set_trace() + #time.sleep(10) + #response2 = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key = 'mlb.l.1328.t.2' AND date = '2015-05-05'") + #response2 = yql.select('fantasysports.teams.roster').where(['team_key', '=', 'mlb.l.1328.t.2'], ['date', '=', '2015-05-05']) + class TestTable(unittest.TestCase): From c796b207844c221eb7679fa53e98fb9fec6209c6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 6 May 2015 09:55:39 +0200 Subject: [PATCH 264/582] updating yahoo_fantasy_sport tests --- tests/tests.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index a313178..52b7b20 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -35,12 +35,16 @@ def test_get_guid(self,): def test_yahoo_fantasy_sport(self,): oauth = YOAuth(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) - #response = yql.select('fantasysports.teams.roster').where(['team_key', '=', 'mlb.l.1328.t.1'], ['date', '=', '2015-05-05']) - response = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key IN ('mlb.l.1328.t.1','mlb.l.1328.t.2') AND date = '2015-05-05'") - pdb.set_trace() - #time.sleep(10) - #response2 = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key = 'mlb.l.1328.t.2' AND date = '2015-05-05'") - #response2 = yql.select('fantasysports.teams.roster').where(['team_key', '=', 'mlb.l.1328.t.2'], ['date', '=', '2015-05-05']) + teams = ('mlb.l.1328.t.1','mlb.l.1328.t.2') + year = '2015-05-05' + for team in teams: + response = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key = '{0}' AND date = '{1}' ".format(team, year)) + if not response.status_code == 200: + return False + + data = response.json() + current_team = data['query']['results']['team'] + print current_team['team_id'],current_team['name'],current_team['number_of_trades'],current_team['number_of_moves'] class TestTable(unittest.TestCase): From caad2932a06c71b7c4a7e479fd46892f5f13b853 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 6 May 2015 10:01:33 +0200 Subject: [PATCH 265/582] doing it like @unpairestgoog :smiley: --- tests/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 52b7b20..2cda54d 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -38,7 +38,8 @@ def test_yahoo_fantasy_sport(self,): teams = ('mlb.l.1328.t.1','mlb.l.1328.t.2') year = '2015-05-05' for team in teams: - response = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key = '{0}' AND date = '{1}' ".format(team, year)) + #response = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key = '{0}' AND date = '{1}' ".format(team, year)) + response = yql.select('fantasysports.teams.roster').where(['team_key','=',team],['date','=',year]) if not response.status_code == 200: return False From 804e9d658e0d8f94622072850130ab93dbcb68a3 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 8 May 2015 13:48:16 +0200 Subject: [PATCH 266/582] header_auth=True added --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 960ca7c..b482ab7 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -86,7 +86,7 @@ def executeQuery(self, payload): self.url = self.private_url if not self.oauth.token_is_valid(): # Refresh token if token has expired self.oauth.refresh_token() - response = self.oauth.session.get(self.url, params= payload) + response = self.oauth.session.get(self.url, params= payload, header_auth=True) else: response = requests.get(self.url, params= payload) From d9d6a872d22c46440a7e1110fea1025531ab9296 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 8 May 2015 14:01:50 +0200 Subject: [PATCH 267/582] adding README.rst --- README.rst | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 README.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..20c59e8 --- /dev/null +++ b/README.rst @@ -0,0 +1,196 @@ +MYQL +==== + +`|Build Status| `_ +`|Documentation Status| `_ `|PyPI +version| `_ `|Join the chat at +https://gitter.im/josuebrunel/myql| `_ +`|Code +Issues| `_ + +MYQL is a Python wrapper of the Yahoo Query Language. + +Yahoo! Query Language Documentation and Support +=============================================== + +- Yahoo! Query Language - http://developer.yahoo.com/yql/ +- Yahoo! Developer Network: http://developer.yahoo.com +- Yahoo! Application Platform - http://developer.yahoo.com/yap/ +- Yahoo! Social APIs - http://developer.yahoo.com/social/ +- Yahoo! QUery Language Console + https://developer.yahoo.com/yql/console/ + +version 1.2 +=========== + +- OpenTable classes +- Access to resources requiring authentication + +version 0.5.6 +============= + +- fetch data +- access to community data +- select data format (xml/json) +- change data source +- filter data + +fixes: +------ + +- fix handling of default and on the fly response format +- fix limit on ***select(...).where(...)*** when no limit value is + passed +- fix limit on ***get(...)*** + +installation +============ + +:: + + $ pip install myql + +how to use +========== + +:: + + >>> import myql + >>> yql = myql.MYQL() + >>> yql.diagnostics = True # To turn diagnostics on + +access to community tables +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + >>> yql = myql.MYQL() + >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') + >>> rep.json() + {u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} + >>> yql.community= True # Setting up access to community + >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') + >>> rep.json() + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} + >>> + +***OR*** + +:: + + >>> import myql + >>> yql = myql.MYQL(community=True) + >>> # do your magic + +changing response format (xml or json) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The response format is by default ***json***. + +:: + + >>> import myql + >>> yql = myql.MYQL(format='xml', community=True) + >>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') + >>> rep.text + u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' + >>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') + >>> rep.json() + {u'query': {u'count': 11, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'55998384', u'name': u'Cuvette-Ouest Department'}, {u'woeid': u'2344968', u'name': u'Cuvette Department'}, {u'woeid': u'2344973', u'name': u'Plateaux District'}, {u'woeid': u'2344974', u'name': u'Sangha'}, {u'woeid': u'2344970', u'name': u'Lekoumou'}, {u'woeid': u'2344975', u'name': u'Pool Department'}, {u'woeid': u'2344971', u'name': u'Likouala Department'}, {u'woeid': u'2344972', u'name': u'Niari Department'}, {u'woeid': u'2344976', u'name': u'Brazzaville'}, {u'woeid': u'2344967', u'name': u'Bouenza Department'}, {u'woeid': u'2344969', u'name': u'Kouilou'}]}, u'created': u'2014-08-27T04:52:38Z'}} + >>> + +Methods +------- + +use(data\_provider\_url) +^^^^^^^^^^^^^^^^^^^^^^^^ + +Changes the data provider + +:: + + >>> yql.use('http://myserver.com/mytables.xml') + +desc(tablename) +^^^^^^^^^^^^^^^ + +Returns table description + +:: + + >>> response = yql.desc('weather.forecast') + >>> response.json() + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'request': {u'select': [{u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'location'}, {u'type': u'xs:string', u'name': u'u'}]}, {u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'woeid'}, {u'type': u'xs:string', u'name': u'u'}]}]}, u'security': u'ANY', u'meta': {u'documentationURL': u'http://developer.yahoo.com/weather/', u'sampleQuery': u'select * from weather.forecast where woeid=2502265', u'description': u'Weather forecast table', u'author': u'Yahoo! Inc'}, u'hash': u'aae78b1462a6a8fbc748aec4cf292767', u'name': u'weather.forecast'}}, u'created': u'2014-08-16T19:31:51Z'}} + >>> + +rawQuery(query) +^^^^^^^^^^^^^^^ + +Allows you to directly type your query + +:: + + >>> response = yql.rawQuery("select * from geo.countries where place='North America'") + >>> # deal with the response + +select(table, fields, limit).where(filters, ...) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Select a table i.e *weather.forecast*. If *table* not provided, it will +use the default table. If there's no such thing as a default table, it +will raise a *NoTableSelectedError* + +***NB*** : A simple select doesn't return any data. Use ***GET*** +instead. + +:: + + >>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) + >>> response.json() + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424775', u'name': u'Canada'}}, u'created': u'2014-08-16T19:04:08Z'}} + >>> ... + >>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', '=', 'Africa']) + >>> rep.json() + {u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T10:52:49Z'}} + >>> + >>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) + >>> rep.json() + {u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T11:22:49Z'}} + >>> + +get(table, fields, limit) +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Same as ***SELECT***, but instead returns data. + +**REMINDER** : Some tables require a **where clause**, therefore +***GET*** won't work on those tables, use *select(...).where(...)* +instead . + +:: + + >>> yql.get('geo.countries', ['name', 'woeid'], 1) + >>> rep.json() + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} + >>> + +Using OAuth to fetch protected resources +======================================== + +:: + + >>> from myql.contrib.auth import YOAuth + >>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. + >>> from myql import MYQL + >>> yql = MYQL(format='xml', oauth=oauth) + >>> response = yql.getGUID('josue_brunel') # Deal with the response + +.. |Build +Status| image:: https://travis-ci.org/josuebrunel/myql.svg?branch=master +.. |Documentation +Status| image:: https://readthedocs.org/projects/myql/badge/?version=latest +.. |PyPI version| image:: https://badge.fury.io/py/myql.svg +.. |Join the chat at +https://gitter.im/josuebrunel/myql| image:: https://badges.gitter.im/Join%20Chat.svg +.. |Code +Issues| image:: https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg From 67f9d429f148956d106f82203d7d0435ad003791 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 06:48:10 +0200 Subject: [PATCH 268/582] upgrading requests version and trying to fix #65 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 83a959f..710e1aa 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ oauthlib==0.7.2 rauth==0.7.1 -requests==2.6.2 requests-oauthlib==0.4.2 wsgiref==0.1.2 +requests==2.7.0 From b642c28fdb40b39a44bc527837a9c7efd742a239 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 08:07:29 +0200 Subject: [PATCH 269/582] updating requirements --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 710e1aa..3bea596 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -oauthlib==0.7.2 -rauth==0.7.1 -requests-oauthlib==0.4.2 +oauthlib>=0.7.2 +rauth>=0.7.1 +requests-oauthlib>=0.4.2 wsgiref==0.1.2 -requests==2.7.0 +requests>=2.7.0 From a5fc3e2895b035abb7294b1af2f8db1a396ba9fd Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 16:20:42 +0200 Subject: [PATCH 270/582] OAuth test run if credentials.jon file exists --- run_tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_tests.sh b/run_tests.sh index 33883f0..1d47142 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,3 +5,6 @@ else fi python -m unittest tests.TestTable$method +if [ -f credentials.json ]; then + python -m unittest tests.TestOAuth +fi From c52f3955731f572e487f42cba8e2df6172c2bd69 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 16:27:03 +0200 Subject: [PATCH 271/582] updating from 1.2.0 to 1.2.1 --- README.md | 19 ++++++++++++------- README.rst | 20 ++++++++++++-------- setup.py | 2 +- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e162e6f..71cc298 100755 --- a/README.md +++ b/README.md @@ -16,22 +16,27 @@ Yahoo! Query Language Documentation and Support * Yahoo! Social APIs - http://developer.yahoo.com/social/ * Yahoo! QUery Language Console https://developer.yahoo.com/yql/console/ -version 1.2 -=========== + +Releases Notes +============== + +v 1.2.1 +------ +* Multiple requests while using OAuth fixed + +v 1.2.0 +------- * OpenTable classes * Access to resources requiring authentication -version 0.5.6 -============= +v 0.5.6 +------------- * fetch data * access to community data * select data format (xml/json) * change data source * filter data - -fixes: ------- * fix handling of default and on the fly response format * fix limit on ***select(...).where(...)*** when no limit value is passed * fix limit on ***get(...)*** diff --git a/README.rst b/README.rst index 20c59e8..34d4e03 100644 --- a/README.rst +++ b/README.rst @@ -20,24 +20,28 @@ Yahoo! Query Language Documentation and Support - Yahoo! QUery Language Console https://developer.yahoo.com/yql/console/ -version 1.2 -=========== +Releases Notes +============== + +v 1.2.1 +------- + +- Multiple requests while using OAuth fixed + +v 1.2.0 +------- - OpenTable classes - Access to resources requiring authentication -version 0.5.6 -============= +v 0.5.6 +------- - fetch data - access to community data - select data format (xml/json) - change data source - filter data - -fixes: ------- - - fix handling of default and on the fly response format - fix limit on ***select(...).where(...)*** when no limit value is passed diff --git a/setup.py b/setup.py index e83ffb6..e18420b 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ def read(fname): setup( name = "myql", - version = "1.2", + version = "1.2.1", description = "Python Wrapper for the Yahoo ! Query Language", long_description = read("README.rst"), author = "Josue Kouka", From 41e0b0c47ba0c71c7cc884c88735de4ea386b404 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 16:33:10 +0200 Subject: [PATCH 272/582] fix typo --- README.md | 2 +- README.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 71cc298..e29d8ff 100755 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Yahoo! Query Language Documentation and Support -Releases Notes +Release Notes ============== v 1.2.1 diff --git a/README.rst b/README.rst index 34d4e03..916a43f 100644 --- a/README.rst +++ b/README.rst @@ -20,8 +20,8 @@ Yahoo! Query Language Documentation and Support - Yahoo! QUery Language Console https://developer.yahoo.com/yql/console/ -Releases Notes -============== +Release Notes +============= v 1.2.1 ------- From a9727e84f5d6d979b6f66e2e7532af85c1110c47 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 16:36:21 +0200 Subject: [PATCH 273/582] dynamic version in setup.py --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e18420b..6f9747a 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ import os from setuptools import setup, find_packages +__version__ = "1.2.1" + #requirements.txt with open('requirements.txt') as f: required = f.read().splitlines() @@ -10,13 +12,13 @@ def read(fname): setup( name = "myql", - version = "1.2.1", + version = __version__, description = "Python Wrapper for the Yahoo ! Query Language", long_description = read("README.rst"), author = "Josue Kouka", author_email = "josuebrunel@gmail.com", url = "https://github.com/josuebrunel/MYQL", - download_url = "https://github.com/josuebrunel/myql/archives/1.2.tar.gz", + download_url = "https://github.com/josuebrunel/myql/archives/{0}tar.gz".format(__version__), keywords = ['myql', 'yql', 'yahoo', 'query', 'language'], packages = find_packages(), classifiers = [ From e18f85e6fb9d959acdf92b15293e4e50e45e17d0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 16:38:51 +0200 Subject: [PATCH 274/582] fix missing README.rst --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index f9bd145..8e5e8a9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include requirements.txt +include README.rst From 26c801f64a1bc3d6aebcd5edb14e6a7a69e3fda2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 17:00:55 +0200 Subject: [PATCH 275/582] fix download_url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6f9747a..0bd9a59 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(fname): author = "Josue Kouka", author_email = "josuebrunel@gmail.com", url = "https://github.com/josuebrunel/MYQL", - download_url = "https://github.com/josuebrunel/myql/archives/{0}tar.gz".format(__version__), + download_url = "https://github.com/josuebrunel/myql/archives/{0}.tar.gz".format(__version__), keywords = ['myql', 'yql', 'yahoo', 'query', 'language'], packages = find_packages(), classifiers = [ From fce64237448c93118475ac95c53acc50032cc73b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 9 May 2015 17:37:56 +0200 Subject: [PATCH 276/582] download url fixed --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0bd9a59..0c1f7e9 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def read(fname): author = "Josue Kouka", author_email = "josuebrunel@gmail.com", url = "https://github.com/josuebrunel/MYQL", - download_url = "https://github.com/josuebrunel/myql/archives/{0}.tar.gz".format(__version__), + download_url = "https://github.com/josuebrunel/myql/archive/{0}.tar.gz".format(__version__), keywords = ['myql', 'yql', 'yahoo', 'query', 'language'], packages = find_packages(), classifiers = [ From 9ae7fe286d2294f01b5d44b4f093b5bea5edfe03 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 10 May 2015 07:22:13 +0200 Subject: [PATCH 277/582] New badges --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e29d8ff..f1feb30 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ MYQL ========= -[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) [![PyPI version](https://badge.fury.io/py/myql.svg)](http://badge.fury.io/py/myql) +[![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) +[![Latest Version](https://pypip.in/version/myql/badge.svg)](https://pypi.python.org/pypi/myql/) +[![Downloads](https://pypip.in/download/myql/badge.svg)](https://pypi.python.org/pypi/myql) +[![Status](https://pypip.in/py_versions/myql/badge.svg)](https://pypi.python.org/pypi/myql) +[![Status](https://pypip.in/implementation/myql/badge.svg)](https://pypi.python.org/pypi/myql) [![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Issues](https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg)](https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql) From a9e300dc2fe0726d5bd8fb1f3bf671361d2c9218 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 10 May 2015 20:40:11 +0200 Subject: [PATCH 278/582] script to convert md file to rst --- convert_to_rst.sh | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100755 convert_to_rst.sh diff --git a/convert_to_rst.sh b/convert_to_rst.sh new file mode 100755 index 0000000..a13921b --- /dev/null +++ b/convert_to_rst.sh @@ -0,0 +1,12 @@ +################################################## +# +# Author : josuebrunel +# Filename : convert_to_rst.sh +# Description : use pandoc to convert md file to rst +# Creation Date : 10-05-2015 +# Last Modified : Sun 10 May 2015 08:39:32 PM CEST +# +################################################## + + +pandoc README.md -t rst -o README.rst diff --git a/setup.py b/setup.py index 0c1f7e9..c2026b5 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ def read(fname): 'License :: OSI Approved :: MIT License' ], platforms=['Any'], - license='BSD', + license='GNU', install_requires = required ) From 73d9d466fb713b5beab1ba63a6dac91ca66d58aa Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 11 May 2015 00:11:21 +0200 Subject: [PATCH 279/582] TestMYQL Added --- myql/myql.py | 4 ++-- tests/tests.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index b482ab7..764f562 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -97,12 +97,12 @@ def clauseFormatter(self, cond): args is a list of ['column', 'operator', 'value'] ''' if cond[1].lower() == 'in': - cond[2] = "{0}".format(cond[2]) + cond[2] = "({0})".format(','.join(map(str,[ "'{0}'".format(e) for e in cond[2] ]))) cond = ' '.join(cond) else: cond[2] = "'{0}'".format(cond[2]) cond = ''.join(cond) - + return cond def buildResponse(self, response): diff --git a/tests/tests.py b/tests/tests.py index 2cda54d..adb4baa 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -11,12 +11,55 @@ from myql.contrib.table import Base, BaseInput from myql.contrib.table import Binder, BinderFunction, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset +from myql.contrib.stockscraper import stockretriever + import readline, rlcompleter readline.parse_and_bind('tab: complete') logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logging.getLogger(__name__) +class TestMYQL(unittest.TestCase): + + def setUp(self,): + self.yql = MYQL(format='json',community=True) + + def tearDown(self): + pass + + def test_raw_query(self,): + response = self.yql.rawQuery('select name, woeid from geo.states where place="Congo"') + self.assertEquals(response.status_code,200) + try: + logging.debug(response.json()) + except Exception,e: + logging.error(e) + + def test_get(self,): + response = self.yql.get('geo.countries', ['name', 'woeid'], 1) + self.assertEquals(response.status_code,200) + try: + logging.debug(response.json()) + except Exception,e: + logging.error(e) + + def test_select(self,): + response = self.yql.select('geo.countries', ['name', 'code', 'woeid']).where(['name', '=', 'Canada']) + self.assertEquals(response.status_code,200) + try: + logging.debug(response.json()) + except Exception,e: + logging.error(e) + + def test_select_in(self,): + response = self.yql.select('yahoo.finance.quotes').where(['symbol','in',("YHOO","AAPL","GOOG","MSFT")]) + self.assertEquals(response.status_code,200) + try: + logging.debug(response.json()) + except Exception,e: + logging.error(e) + + class TestOAuth(unittest.TestCase): def setUp(self,): @@ -47,6 +90,12 @@ def test_yahoo_fantasy_sport(self,): current_team = data['query']['results']['team'] print current_team['team_id'],current_team['name'],current_team['number_of_trades'],current_team['number_of_moves'] +class TestStockParser(unittest.TestCase): + + def test_get_current_info(self,): + + data = stockretriever.get_current_info(["YHOO","AAPL","GOOG"]) + logging.debug(data) class TestTable(unittest.TestCase): From 8a5bc0709dabf109cd7c601aa87270b00c2a21ec Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 11 May 2015 00:11:46 +0200 Subject: [PATCH 280/582] Run common MYQL tests added --- run_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run_tests.sh b/run_tests.sh index 1d47142..97f7322 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,6 +4,7 @@ else method='' fi +python -m unittest tests.TestMYQL$method python -m unittest tests.TestTable$method if [ -f credentials.json ]; then python -m unittest tests.TestOAuth From 29678eb74e4e5fc19558e5335571b0df4948c4e6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 11 May 2015 00:24:58 +0200 Subject: [PATCH 281/582] pretty_json function added --- tests/tests.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index adb4baa..672cbfc 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,4 +1,4 @@ -import os, logging, time +import os, logging, time, json import pdb import unittest from xml.dom import minidom @@ -19,6 +19,11 @@ logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logging.getLogger(__name__) + +def pretty_json(data): + data = json.loads(data) + return json.dumps(data, indent=4, sort_keys=False) + class TestMYQL(unittest.TestCase): def setUp(self,): @@ -31,7 +36,7 @@ def test_raw_query(self,): response = self.yql.rawQuery('select name, woeid from geo.states where place="Congo"') self.assertEquals(response.status_code,200) try: - logging.debug(response.json()) + logging.debug(pretty_json(response.content)) except Exception,e: logging.error(e) @@ -39,7 +44,7 @@ def test_get(self,): response = self.yql.get('geo.countries', ['name', 'woeid'], 1) self.assertEquals(response.status_code,200) try: - logging.debug(response.json()) + logging.debug(pretty_json(response.content)) except Exception,e: logging.error(e) @@ -47,7 +52,7 @@ def test_select(self,): response = self.yql.select('geo.countries', ['name', 'code', 'woeid']).where(['name', '=', 'Canada']) self.assertEquals(response.status_code,200) try: - logging.debug(response.json()) + logging.debug(pretty_json(response.content)) except Exception,e: logging.error(e) @@ -55,7 +60,7 @@ def test_select_in(self,): response = self.yql.select('yahoo.finance.quotes').where(['symbol','in',("YHOO","AAPL","GOOG","MSFT")]) self.assertEquals(response.status_code,200) try: - logging.debug(response.json()) + logging.debug(pretty_json(response.content)) except Exception,e: logging.error(e) From bee8810b80f0be69c9333ac06d914cf96dad2228 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 11 May 2015 00:29:33 +0200 Subject: [PATCH 282/582] fixing issue with stockscraper --- tests/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 672cbfc..74e015f 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -11,7 +11,7 @@ from myql.contrib.table import Base, BaseInput from myql.contrib.table import Binder, BinderFunction, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset -from myql.contrib.stockscraper import stockretriever +#from myql.contrib.stockscraper import stockretriever import readline, rlcompleter readline.parse_and_bind('tab: complete') @@ -95,12 +95,12 @@ def test_yahoo_fantasy_sport(self,): current_team = data['query']['results']['team'] print current_team['team_id'],current_team['name'],current_team['number_of_trades'],current_team['number_of_moves'] -class TestStockParser(unittest.TestCase): - - def test_get_current_info(self,): - - data = stockretriever.get_current_info(["YHOO","AAPL","GOOG"]) - logging.debug(data) +#class TestStockParser(unittest.TestCase): +# +# def test_get_current_info(self,): +# +# data = stockretriever.get_current_info(["YHOO","AAPL","GOOG"]) +# logging.debug(data) class TestTable(unittest.TestCase): From fb3a457e3ddfbdf2254ddbb1ecebdac68e03c345 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 00:58:08 +0200 Subject: [PATCH 283/582] json result sort key = True --- tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 672cbfc..c2b079a 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -22,7 +22,7 @@ def pretty_json(data): data = json.loads(data) - return json.dumps(data, indent=4, sort_keys=False) + return json.dumps(data, indent=4, sort_keys=True) class TestMYQL(unittest.TestCase): From 4b3f65ddb263784be310f0a2239095114c26725c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 01:03:33 +0200 Subject: [PATCH 284/582] #68 : get_current_info OK --- myql/contrib/stockscraper/__init__.py | 5 +++++ myql/contrib/stockscraper/stockretriever.py | 9 +++++++++ tests/tests.py | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 myql/contrib/stockscraper/__init__.py create mode 100644 myql/contrib/stockscraper/stockretriever.py diff --git a/myql/contrib/stockscraper/__init__.py b/myql/contrib/stockscraper/__init__.py new file mode 100644 index 0000000..b2b86eb --- /dev/null +++ b/myql/contrib/stockscraper/__init__.py @@ -0,0 +1,5 @@ +#import stockretriever + +__all__ = [ + 'stockretriever' +] diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py new file mode 100644 index 0000000..9e341e6 --- /dev/null +++ b/myql/contrib/stockscraper/stockretriever.py @@ -0,0 +1,9 @@ +import pdb +from myql.myql import MYQL + +def get_current_info(symbolList, columns=None, format='json'): + """ + """ + yql = MYQL(format=format, community=True) + response = yql.select('yahoo.finance.quotes',columns).where(['symbol','in',symbolList]) + return response diff --git a/tests/tests.py b/tests/tests.py index c2b079a..edd3942 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -100,7 +100,7 @@ class TestStockParser(unittest.TestCase): def test_get_current_info(self,): data = stockretriever.get_current_info(["YHOO","AAPL","GOOG"]) - logging.debug(data) + logging.debug(pretty_json(data.content)) class TestTable(unittest.TestCase): From 58e4b3a13b7c8025ebc8851419bccd91fd192023 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 01:17:44 +0200 Subject: [PATCH 285/582] Improving test script --- run_tests.sh | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 97f7322..6ff6607 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,11 +1,23 @@ -if [ ! -z $1 ]; then - method=".$1" +if [ ! -z $1 ]; then + suite=".$1" +else + suite="all" +fi + +if [ ! -z $2 ]; then + method=".$2" else method='' fi -python -m unittest tests.TestMYQL$method -python -m unittest tests.TestTable$method -if [ -f credentials.json ]; then - python -m unittest tests.TestOAuth +if [ $suite != "all" ];then + python -m unittest tests$suite$method +else + python -m unittest tests.TestMYQL$method + python -m unittest tests.TestStockParser + python -m unittest tests.TestTable$method + + if [ -f credentials.json ]; then + python -m unittest tests.TestOAuth + fi fi From 14b0d1508d0c1027259d74a759336da9925235ed Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 01:30:30 +0200 Subject: [PATCH 286/582] #68: get news feed tests added --- tests/tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index edd3942..b2aa091 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -97,10 +97,17 @@ def test_yahoo_fantasy_sport(self,): class TestStockParser(unittest.TestCase): - def test_get_current_info(self,): + def get_current_info(self,): data = stockretriever.get_current_info(["YHOO","AAPL","GOOG"]) logging.debug(pretty_json(data.content)) + self.assertEquals(data.status_code,200) + + def get_news_feed(self,): + data = stockretriever.get_news_feed('YHOO') + logging.debug(pretty_json(data.content)) + self.assertEquals(data.status_code,200) + class TestTable(unittest.TestCase): From 5b061fabcd6d19869a2a5ed4d8c085c48d5bde0c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 01:30:49 +0200 Subject: [PATCH 287/582] #68: get news feed OK --- myql/contrib/stockscraper/stockretriever.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index 9e341e6..bf861cd 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -2,8 +2,17 @@ from myql.myql import MYQL def get_current_info(symbolList, columns=None, format='json'): - """ + """get_current_info() uses the yahoo.finance.quotes datatable to get all of the stock information presented in the main table on a typical stock page + and a bunch of data from the key statistics page. """ yql = MYQL(format=format, community=True) response = yql.select('yahoo.finance.quotes',columns).where(['symbol','in',symbolList]) return response + +def get_news_feed(symbol,format='json'): + """get_news_feed() uses the rss data table to get rss feeds under the Headlines and Financial Blogs headings on a typical stock page. + """ + yql = MYQL(format=format, community=True) + rss_url='http://finance.yahoo.com/rss/headline?s={0}'.format(symbol) + response = yql.select('rss',['title','link','description'],limit=2).where(['url','=',rss_url]) + return response From 31ce6d3df469b618aa9cff07f55300901321f001 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 01:40:43 +0200 Subject: [PATCH 288/582] #68: get_historical_info OK --- myql/contrib/stockscraper/stockretriever.py | 8 ++++++++ tests/tests.py | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index bf861cd..ce12c00 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -16,3 +16,11 @@ def get_news_feed(symbol,format='json'): rss_url='http://finance.yahoo.com/rss/headline?s={0}'.format(symbol) response = yql.select('rss',['title','link','description'],limit=2).where(['url','=',rss_url]) return response + +def get_historical_info(symbol,limit=5, format='json'): + """get_historical_info() uses the csv datatable to retrieve all available historical data on a typical historical prices page + """ + yql = MYQL(format=format) + historical_url = 'http://ichart.finance.yahoo.com/table.csv?s={0}'.format(symbol) + response = yql.select('csv',limit=limit).where(['url','=',historical_url],['columns','=','Date,Open,High,Low,Close,Volume,AdjClose']) + return response diff --git a/tests/tests.py b/tests/tests.py index b2aa091..a6b3bb7 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -107,7 +107,11 @@ def get_news_feed(self,): data = stockretriever.get_news_feed('YHOO') logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) - + + def get_historical_info(self,): + data = stockretriever.get_historical_info('YHOO') + logging.debug(pretty_json(data.content)) + self.assertEquals(data.status_code,200) class TestTable(unittest.TestCase): From eb79036f2f2390db0867502ea56983ffb604bdc4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 01:46:41 +0200 Subject: [PATCH 289/582] updating test script --- run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 6ff6607..295a6db 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -13,9 +13,9 @@ fi if [ $suite != "all" ];then python -m unittest tests$suite$method else - python -m unittest tests.TestMYQL$method + python -m unittest tests.TestMYQL python -m unittest tests.TestStockParser - python -m unittest tests.TestTable$method + python -m unittest tests.TestTable if [ -f credentials.json ]; then python -m unittest tests.TestOAuth From dced94b0282eb121a880ad1c2ab073af79a0fe7e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 01:55:13 +0200 Subject: [PATCH 290/582] trying to fix tests run for TestStockScraper --- tests/tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index a6b3bb7..d91b615 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -96,7 +96,13 @@ def test_yahoo_fantasy_sport(self,): print current_team['team_id'],current_team['name'],current_team['number_of_trades'],current_team['number_of_moves'] class TestStockParser(unittest.TestCase): - + + def setUp(self,): + pass + + def tearDown(self): + pass + def get_current_info(self,): data = stockretriever.get_current_info(["YHOO","AAPL","GOOG"]) @@ -113,6 +119,7 @@ def get_historical_info(self,): logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) + class TestTable(unittest.TestCase): def setUp(self,): From 211f27cd6730e51420c3bafd9fea6ec515e10322 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 02:02:46 +0200 Subject: [PATCH 291/582] updating tests --- run_tests.sh | 2 +- tests/tests.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 295a6db..2180e3b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -14,7 +14,7 @@ if [ $suite != "all" ];then python -m unittest tests$suite$method else python -m unittest tests.TestMYQL - python -m unittest tests.TestStockParser + python -m unittest tests.TestStockParser.{get_current_info,get_news_feed,get_historical_info} python -m unittest tests.TestTable if [ -f credentials.json ]; then diff --git a/tests/tests.py b/tests/tests.py index d91b615..06d53ee 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -88,6 +88,7 @@ def test_yahoo_fantasy_sport(self,): for team in teams: #response = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key = '{0}' AND date = '{1}' ".format(team, year)) response = yql.select('fantasysports.teams.roster').where(['team_key','=',team],['date','=',year]) + self.assertEquals(response.status_code,200) if not response.status_code == 200: return False @@ -95,6 +96,7 @@ def test_yahoo_fantasy_sport(self,): current_team = data['query']['results']['team'] print current_team['team_id'],current_team['name'],current_team['number_of_trades'],current_team['number_of_moves'] + class TestStockParser(unittest.TestCase): def setUp(self,): From e98906a0804ed994bf803c18d998a47b583b3d49 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 07:41:37 +0200 Subject: [PATCH 292/582] #68: Get options info added --- tests/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 06d53ee..2011423 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -121,6 +121,11 @@ def get_historical_info(self,): logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) + def get_options_info(self,): + data = stockretriever.get_options_info('YHOO') + logging.debug(pretty_json(data.content)) + self.assertEquals(data.status_code,200) + class TestTable(unittest.TestCase): From d3fe026ad7b7bc5afcc004f8df5aae612a82559a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 08:40:55 +0200 Subject: [PATCH 293/582] #68: get options info added --- myql/contrib/stockscraper/stockretriever.py | 8 ++++++++ myql/myql.py | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index ce12c00..b23f5b9 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -24,3 +24,11 @@ def get_historical_info(symbol,limit=5, format='json'): historical_url = 'http://ichart.finance.yahoo.com/table.csv?s={0}'.format(symbol) response = yql.select('csv',limit=limit).where(['url','=',historical_url],['columns','=','Date,Open,High,Low,Close,Volume,AdjClose']) return response + +def get_options_info(symbol, items=[], expiration='', format=format): + """get_options_data() uses the yahoo.finance.options table to retrieve call and put options from the options page. + """ + yql = MYQL(format=format,community=True) + response = yql.select('yahoo.finance.options',items).where(['symbol','=',symbol],[] if not expiration else ['expiration','=',expiration]) + return response + diff --git a/myql/myql.py b/myql/myql.py index 764f562..5ffde1d 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -204,8 +204,9 @@ def where(self, *args): clause = [] self._query += ' where ' for x in args: - x = self.clauseFormatter(x) - clause.append(x) + if x: + x = self.clauseFormatter(x) + clause.append(x) self._query += ' and '.join(clause) From e14638bf84f7dff911097f6aec450833e531e5d0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 10:09:50 +0200 Subject: [PATCH 294/582] fix tests imports --- tests/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index ecadf74..bd6d96e 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,4 @@ -from tests import * +from tests import TestMYQL +from tests import TestStockParser +from tests import TestTable +from tests import TestOAuth From 36f52962fb5e762dc24de105c9a17a2e97ec187c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 10:39:17 +0200 Subject: [PATCH 295/582] #1 : v1.2.2 release notes --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index e29d8ff..b0a3753 100755 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ Yahoo! Query Language Documentation and Support Release Notes ============== +v 1.2.2 ( developpement ) +------- +* Issue with **IN** condition in **where** clause fixed +* Issue when passing an **[]/()** in a **where** claused fixed +* Import of StockParser from Gurchet Rai https://github.com/gurch101/StockScraper OK + v 1.2.1 ------ * Multiple requests while using OAuth fixed From 95f424fb5aa55054b01d449b2b2a822a900c5ea7 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 11:04:14 +0200 Subject: [PATCH 296/582] #68 : Get index summary tests added --- tests/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 2011423..3a5e1d5 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -126,6 +126,11 @@ def get_options_info(self,): logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) + def get_index_summary(self,): + data = stockretriever.get_index_summary('GOOG',('Volume','Change')) + logging.debug(pretty_json(data.content)) + self.assertEquals(data.status_code,200) + class TestTable(unittest.TestCase): From 5b31af1c2f7adde57b1887b969e9482f814bdcb8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 11:04:38 +0200 Subject: [PATCH 297/582] #68: get index summary OK --- myql/contrib/stockscraper/stockretriever.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index b23f5b9..3174e01 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -1,3 +1,8 @@ +"""This module isn't mine, it's at 99% inspired from https://github.com/gurch101/StockScraper written by Gurchet Rai. +So all rigts reserved to Gurchet Rai. +Documentation http://www.gurchet-rai.net/dev/yahoo-finance-yql +""" + import pdb from myql.myql import MYQL @@ -32,3 +37,9 @@ def get_options_info(symbol, items=[], expiration='', format=format): response = yql.select('yahoo.finance.options',items).where(['symbol','=',symbol],[] if not expiration else ['expiration','=',expiration]) return response +def get_index_summary(symbol, items=[],format='json'): + """ + """ + yql = MYQL(format=format, community=True) + response = yql.select('yahoo.finance.quoteslist',items).where(['symbol','=',symbol]) + return response From 851a84f1d63650615660d02edf9fdceb95fe7eef Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 11:18:49 +0200 Subject: [PATCH 298/582] #68: tests and get index industry OK --- myql/contrib/stockscraper/stockretriever.py | 7 +++++++ tests/tests.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index 3174e01..d82efb1 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -43,3 +43,10 @@ def get_index_summary(symbol, items=[],format='json'): yql = MYQL(format=format, community=True) response = yql.select('yahoo.finance.quoteslist',items).where(['symbol','=',symbol]) return response + +def get_industry_index(index_id,items=[],format='json'): + """retrieves all symbols that belong to an industry. + """ + yql = MYQL(community=True,format=format) + response = yql.select('yahoo.finance.industry',items).where(['id','=',index_id]) + return response diff --git a/tests/tests.py b/tests/tests.py index 3a5e1d5..8bbba91 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -131,6 +131,11 @@ def get_index_summary(self,): logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) + def get_industry_index(self,): + data = stockretriever.get_industry_index(112) + logging.debug(pretty_json(data.content)) + self.assertEquals(data.status_code,200) + class TestTable(unittest.TestCase): From 3b0c6aab29190e127cb7cfe5ebf37a77870ecfbe Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 11:24:47 +0200 Subject: [PATCH 299/582] #68 updating test script --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 2180e3b..5619da3 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -14,7 +14,7 @@ if [ $suite != "all" ];then python -m unittest tests$suite$method else python -m unittest tests.TestMYQL - python -m unittest tests.TestStockParser.{get_current_info,get_news_feed,get_historical_info} + python -m unittest tests.TestStockParser.{get_current_info,get_news_feed,get_historical_info,get_options_info,get_index_summary,get_industry_index} python -m unittest tests.TestTable if [ -f credentials.json ]; then From 7929b4b601fa2ea0d291e4d8229588801d99298d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 11:40:49 +0200 Subject: [PATCH 300/582] updating tests --- tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 8bbba91..55d2218 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -57,7 +57,7 @@ def test_select(self,): logging.error(e) def test_select_in(self,): - response = self.yql.select('yahoo.finance.quotes').where(['symbol','in',("YHOO","AAPL","GOOG","MSFT")]) + response = self.yql.select('yahoo.finance.quotes').where(['symbol','in',("YHOO","AAPL","GOOG")]) self.assertEquals(response.status_code,200) try: logging.debug(pretty_json(response.content)) From 95d6a1059aae037b6886fde99ee70d05495328db Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 11:54:50 +0200 Subject: [PATCH 301/582] fixing development typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b0a3753..a69eb1a 100755 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Yahoo! Query Language Documentation and Support Release Notes ============== -v 1.2.2 ( developpement ) +v 1.2.2 ( development ) ------- * Issue with **IN** condition in **where** clause fixed * Issue when passing an **[]/()** in a **where** claused fixed From a9b6f1a70637f6f44cb0e3c694383d203ed4dd17 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 12:04:58 +0200 Subject: [PATCH 302/582] insert test added --- tests/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 55d2218..3c05580 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -64,6 +64,14 @@ def test_select_in(self,): except Exception,e: logging.error(e) + def test_insert(self,): + response = self.yql.insert('bit.ly.shorten',[('login','o_3v1pqulij6'),('apiKey','R_ee49daf1a1e6492ba56839b641845223'),('longUrl','http://josuebrunel.org')]) + self.assertEquals(response.status_code,200) + try: + logging.debug(pretty_json(response.content)) + except Exception,e: + logging.error(e) + class TestOAuth(unittest.TestCase): From cf44f424ee341950e321d655725fe30b9218010a Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 12 May 2015 12:37:50 +0200 Subject: [PATCH 303/582] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9b3f873..8fce556 100755 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Yahoo! Query Language Documentation and Support Release Notes ============== -v 1.2.2 ( developpement ) +v 1.2.2 ( development ) ------- * Issue with **IN** condition in **where** clause fixed * Issue when passing an **[]/()** in a **where** claused fixed From 06268192a65ce54793bec6e7ff9ce3a664dc4c04 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 15:20:12 +0200 Subject: [PATCH 304/582] #67 : insert test updated --- tests/tests.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 3c05580..25aa2e4 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -65,14 +65,16 @@ def test_select_in(self,): logging.error(e) def test_insert(self,): - response = self.yql.insert('bit.ly.shorten',[('login','o_3v1pqulij6'),('apiKey','R_ee49daf1a1e6492ba56839b641845223'),('longUrl','http://josuebrunel.org')]) - self.assertEquals(response.status_code,200) + #response = self.yql.insert('bit.ly.shorten',('login','apiKey','longUrl'),('o_3v1pqulij6','R_ee49daf1a1e6492ba56839b641845223','http://josuebrunel.org')) + response = self.yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) try: logging.debug(pretty_json(response.content)) except Exception,e: + logging.error(response.content) logging.error(e) - - + + self.assertEquals(response.status_code,200) + class TestOAuth(unittest.TestCase): def setUp(self,): From 2a153ae2b29b41497779e9861d6d7c67bcffc4e2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 15:20:55 +0200 Subject: [PATCH 305/582] #67: insert method OK --- myql/myql.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/myql/myql.py b/myql/myql.py index 5ffde1d..026787f 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -1,3 +1,4 @@ +import logging import requests from contrib.auth import YOAuth import errors @@ -7,6 +8,9 @@ __author__ = 'Josue Kouka' __email__ = 'josuebrunel@gmail.com' +logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") +logger = logging.getLogger(__name__) + class MYQL(object): '''Yet another Python Yahoo! Query Language Wrapper Attributes: @@ -193,6 +197,20 @@ def select(self, table=None, items=[], limit=''): return self + ## INSERT + def insert(self, table,items, values): + """This method allows to insert data into table + >>> yql.insert('bi.ly.shorten',('login','apiKey','longUrl'),('YOUR LOGIN','YOUR API KEY','YOUR LONG URL')) + """ + values = ["'{0}'".format(e) for e in values] + self._query = "INSERT INTO {0} ({1}) VALUES ({2})".format(table,','.join(items),','.join(values)) + logger.debug(self._query) + payload = self.payloadBuilder(self._query) + logger.debug(payload) + response = self.executeQuery(payload) + + return response + ## WHERE def where(self, *args): ''' This method simulates a where condition. Use as follow: From 4cd058b6281ede1a088e31d727d13e68e65015a1 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 15:26:38 +0200 Subject: [PATCH 306/582] #67 updating tests --- tests/tests.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 25aa2e4..b99376c 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -34,38 +34,37 @@ def tearDown(self): def test_raw_query(self,): response = self.yql.rawQuery('select name, woeid from geo.states where place="Congo"') - self.assertEquals(response.status_code,200) try: logging.debug(pretty_json(response.content)) except Exception,e: logging.error(e) + self.assertEquals(response.status_code,200) def test_get(self,): response = self.yql.get('geo.countries', ['name', 'woeid'], 1) - self.assertEquals(response.status_code,200) try: logging.debug(pretty_json(response.content)) except Exception,e: logging.error(e) + self.assertEquals(response.status_code,200) def test_select(self,): response = self.yql.select('geo.countries', ['name', 'code', 'woeid']).where(['name', '=', 'Canada']) - self.assertEquals(response.status_code,200) try: logging.debug(pretty_json(response.content)) except Exception,e: logging.error(e) + self.assertEquals(response.status_code,200) def test_select_in(self,): response = self.yql.select('yahoo.finance.quotes').where(['symbol','in',("YHOO","AAPL","GOOG")]) - self.assertEquals(response.status_code,200) try: logging.debug(pretty_json(response.content)) except Exception,e: logging.error(e) + self.assertEquals(response.status_code,200) def test_insert(self,): - #response = self.yql.insert('bit.ly.shorten',('login','apiKey','longUrl'),('o_3v1pqulij6','R_ee49daf1a1e6492ba56839b641845223','http://josuebrunel.org')) response = self.yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) try: logging.debug(pretty_json(response.content)) @@ -74,6 +73,16 @@ def test_insert(self,): logging.error(e) self.assertEquals(response.status_code,200) + + def test_update(self,): + response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)) + try: + logging.debug(pretty_json(response.content)) + except Exception,e: + logging.error(response.content) + logging.error(e) + + self.assertEquals(response.status_code,200) class TestOAuth(unittest.TestCase): From 30cab1a27e7c6027ba8d63e468061fbeab37bb0f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 15:58:26 +0200 Subject: [PATCH 307/582] #67: update test OK --- tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index b99376c..c8357c3 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -74,8 +74,8 @@ def test_insert(self,): self.assertEquals(response.status_code,200) - def test_update(self,): - response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)) + def test_update(self,): + response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) try: logging.debug(pretty_json(response.content)) except Exception,e: From 9542789c01a6dddd6d253def44b156973a7ca139 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 15:58:48 +0200 Subject: [PATCH 308/582] #67: update method OK --- myql/myql.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 026787f..b28979b 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -50,7 +50,7 @@ def payloadBuilder(self, query, format='json'): '''Build the payload''' if self.community : query = self.community_data + query # access to community data tables - + logger.debug(query) payload = { 'q' : query, 'callback' : '', #This is not javascript @@ -63,7 +63,7 @@ def payloadBuilder(self, query, format='json'): payload['crossProduct'] = self.crossProduct self._payload = payload - + logger.debug(payload) return payload def rawQuery(self, query, format='', pretty=False): @@ -204,17 +204,27 @@ def insert(self, table,items, values): """ values = ["'{0}'".format(e) for e in values] self._query = "INSERT INTO {0} ({1}) VALUES ({2})".format(table,','.join(items),','.join(values)) - logger.debug(self._query) payload = self.payloadBuilder(self._query) - logger.debug(payload) response = self.executeQuery(payload) return response + ## UPDATE + def update(self, table, items, values): + """Updates a YQL Table + >>> yql.update('yql.storage',['value'],['https://josuebrunel.orkg']).where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) + """ + self.table = table + self._limit = None + items_values = ','.join(["{0} = '{1}'".format(k,v) for k,v in zip(items,values)]) + self._query = "UPDATE {0} SET {1}".format(self.table, items_values) + + return self + ## WHERE def where(self, *args): ''' This method simulates a where condition. Use as follow: - >>>yql.select('mytable').where(['name', '=', 'alain'], ['location', '!=', 'paris']) + >>> yql.select('mytable').where(['name', '=', 'alain'], ['location', '!=', 'paris']) ''' if not self.table: raise errors.NoTableSelectedError('No Table Selected') From 26c1a7156b801003b0d950fe77cd1b853d5615fb Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 16:18:33 +0200 Subject: [PATCH 309/582] #67: IUD tests updated. Saving store keys in json file --- tests/tests.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index c8357c3..130942b 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -24,10 +24,23 @@ def pretty_json(data): data = json.loads(data) return json.dumps(data, indent=4, sort_keys=True) +def json_write_data(json_data, filename): + with open(filename, 'w') as fp: + json.dump(json_data, fp, indent=4, encoding= 'utf-8', sort_keys=True) + return True + return False + +def json_get_data(filename): + with open(filename) as fp: + json_data = json.laod(fp) + return json_data + + class TestMYQL(unittest.TestCase): def setUp(self,): self.yql = MYQL(format='json',community=True) + self.insert_result = None def tearDown(self): pass @@ -68,6 +81,9 @@ def test_insert(self,): response = self.yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) try: logging.debug(pretty_json(response.content)) + data = response.json()['query']['results']['inserted'] + logging.debug(data) + json_write_data(data,'yql_storage.json') except Exception,e: logging.error(response.content) logging.error(e) @@ -75,7 +91,8 @@ def test_insert(self,): self.assertEquals(response.status_code,200) def test_update(self,): - response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) + json_data = json_get_data('yql_storage.json') + response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=',json_data['update']]) try: logging.debug(pretty_json(response.content)) except Exception,e: From 98fa1095e61858719548f95c1a27b782386242a2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 16:30:20 +0200 Subject: [PATCH 310/582] #67: fix typo on json.load --- tests/tests.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 130942b..92afc55 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -32,7 +32,7 @@ def json_write_data(json_data, filename): def json_get_data(filename): with open(filename) as fp: - json_data = json.laod(fp) + json_data = json.load(fp) return json_data @@ -90,6 +90,17 @@ def test_insert(self,): self.assertEquals(response.status_code,200) + def test_check_insert(self,): + json_data = json_get_data('yql_storage.json') + response = self.yql.select('yql.storage').where(['name','=',json_data['select']]) + try: + logging.debug(pretty_json(response.content)) + except Exception,e: + logging.error(response.content) + logging.error(e) + + self.assertEquals(response.status_code,200) + def test_update(self,): json_data = json_get_data('yql_storage.json') response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=',json_data['update']]) From 57d566de9e2fe5d8bb5fc747e7113089a147418d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 16:34:38 +0200 Subject: [PATCH 311/582] #67: delete method tests OK --- tests/tests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 92afc55..82da9c2 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -111,6 +111,19 @@ def test_update(self,): logging.error(e) self.assertEquals(response.status_code,200) + + def test_delete(self,): + json_data = json_get_data('yql_storage.json') + response = self.yql.delete('yql.storage').where(['name','=',json_data['update']]) + try: + logging.debug(pretty_json(response.content)) + except Exception,e: + logging.error(response.content) + logging.error(e) + + self.assertEquals(response.status_code,200) + + class TestOAuth(unittest.TestCase): From f5072b643f99731df3a67bbf4cd2012be2597c16 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 16:41:56 +0200 Subject: [PATCH 312/582] #67 : delete OK --- myql/myql.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/myql/myql.py b/myql/myql.py index b28979b..d1978d1 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -221,6 +221,17 @@ def update(self, table, items, values): return self + ## DELETE + def delete(self, table): + """Deletes record in table + >>> yql.delete('yql.storage').where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) + """ + self.table = table + self._limit = None + self._query = "DELETE FROM {0}".format(self.table) + + return self + ## WHERE def where(self, *args): ''' This method simulates a where condition. Use as follow: From 3dbab365fea34bbad040c60edb0ccb3126fb06df Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 17:02:58 +0200 Subject: [PATCH 313/582] removing dummy try except --- myql/myql.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index d1978d1..19a3249 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -183,17 +183,14 @@ def select(self, table=None, items=[], limit=''): >>> yql.select('geo.countries', limit=5) >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) ''' - try: - self.table = table - if not items: - items = ['*'] - self._query = "select {1} from {0} ".format(self.table, ','.join(items)) - try: #Checking wether a limit is set or not - self._limit = limit - except Exception, e: - pass + self.table = table + if not items: + items = ['*'] + self._query = "select {1} from {0} ".format(self.table, ','.join(items)) + try: #Checking wether a limit is set or not + self._limit = limit except Exception, e: - print(e) + pass return self From fb6bb89ec54231743d4904af60c62c8012079cba Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 17:04:59 +0200 Subject: [PATCH 314/582] removing dummy try except for get method --- myql/myql.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 19a3249..779a9db 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -158,21 +158,18 @@ def get(self, table=None, items=[], limit=''): '''Just a select which returns a response >>> yql.get("geo.countries', ['name', 'woeid'], 5") ''' - try: - self.table = table - if not items: + self.table = table + if not items: items = ['*'] - self._query = "select {1} from {0} ".format(self.table, ','.join(items)) - if limit: + self._query = "select {1} from {0} ".format(self.table, ','.join(items)) + if limit: self._query += "limit {0}".format(limit) - if not self.table : + if not self.table : raise errors.NoTableSelectedError('Please select a table') - - payload = self.payloadBuilder(self._query) - response = self.executeQuery(payload) - except Exception, e: - print(e.text) + + payload = self.payloadBuilder(self._query) + response = self.executeQuery(payload) return response From a4827be7726ce474b3c96d4a94f2211aaa147278 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 12 May 2015 17:16:19 +0200 Subject: [PATCH 315/582] #1 : updating release notes --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a69eb1a..1ed6f26 100755 --- a/README.md +++ b/README.md @@ -23,9 +23,12 @@ Release Notes v 1.2.2 ( development ) ------- -* Issue with **IN** condition in **where** clause fixed -* Issue when passing an **[]/()** in a **where** claused fixed -* Import of StockParser from Gurchet Rai https://github.com/gurch101/StockScraper OK +* Fixed issue with **IN** condition in **where** clause +* Fixed issue when passing an empty list/tuple (**[]/()**) in a **where** clause +* Import of StockParser from Gurchet Rai https://github.com/gurch101/StockScraper OK [#68](https://github.com/josuebrunel/myql/issues/68) +* Insert, Update, Delete methods added [#67](https://github.com/josuebrunel/myql/issues/67) +* Dummy *try/except* removed from main module + v 1.2.1 ------ From cd00ffddb1216ee8f78c60c51a826fb192f18781 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 14 May 2015 15:51:05 +0200 Subject: [PATCH 316/582] fix #67 : insert, update, delete OK --- README.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e48c15..1fc5977 100755 --- a/README.md +++ b/README.md @@ -166,12 +166,70 @@ Same as ***SELECT***, but instead returns data. **REMINDER** : Some tables require a **where clause**, therefore ***GET*** won't work on those tables, use *select(...).where(...)* instead . ```python ->>> yql.get('geo.countries', ['name', 'woeid'], 1) +>>> rep = yql.get('geo.countries', ['name', 'woeid'], 1) >>> rep.json() {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} >>> ``` +####insert(table, (field1, field2, ..., fieldN),(value1, value2, ..., valueN)) +Insert values into a table. Arguments 2 and 3 may be **tuples** or **list**. + +```python +>>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) +>>> response.json() # result prettyfied just for the example +{ + "query": { + "count": 1, + "created": "2015-05-14T13:25:56Z", + "lang": "en-US", + "results": { + "inserted": { + "execute": "store://KkkC5xDw4v32IcWWSQ4YRe", + "select": "store://Zc5LHXcmYM7XBfSbo9tzFL", + "update": "store://Rqb5fbQyDvrfHJiClWnZ6q" + } + } + } +} +``` + +####update(table,[field1, ..., fieldN],[value1, ..., ...valueN]).where(filters, ...) +Update fields values. This method __is always followed by ***where()***__. Arguments 2 and 3 may be **tuples** or **list**. + +```python +>>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=',json_data['update']]) +>>> response.json() # result prettyfied just for the example +{ + "query": { + "count": 1, + "created": "2015-05-14T13:32:52Z", + "lang": "en-US", + "results": { + "success": "Updated store://KkkC5xDw4v32IcWWSQ4YRe" + } + } +} +``` + +####delete(table).where(filters, ...) +Delete records +```python +>>> response = self.yql.delete('yql.storage').where(['name','=',json_data['update']]) +>>> response.json() # result prettyfied just for the example +{ + "query": { + "count": 1, + "created": "2015-05-14T13:38:28Z", + "lang": "en-US", + "results": { + "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" + } + } +} + +``` + Using OAuth to fetch protected resources ========================================= From 06491b2e6fb3448d3f249780a9a4bed628f3fe6a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 14 May 2015 16:12:31 +0200 Subject: [PATCH 317/582] #1: Documentation updated --- README.md | 4 ++-- docs/index.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ docs/myql.md | 30 ++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1fc5977..efcf442 100755 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ Insert values into a table. Arguments 2 and 3 may be **tuples** or **list**. Update fields values. This method __is always followed by ***where()***__. Arguments 2 and 3 may be **tuples** or **list**. ```python ->>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=',json_data['update']]) +>>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) >>> response.json() # result prettyfied just for the example { "query": { @@ -215,7 +215,7 @@ Update fields values. This method __is always followed by ***where()***__. Argum ####delete(table).where(filters, ...) Delete records ```python ->>> response = self.yql.delete('yql.storage').where(['name','=',json_data['update']]) +>>> response = self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) >>> response.json() # result prettyfied just for the example { "query": { diff --git a/docs/index.md b/docs/index.md index eb8c93b..c38b56d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -137,6 +137,64 @@ Same as ***SELECT***, but instead returns data. >>> ``` +####insert(table, (field1, field2, ..., fieldN),(value1, value2, ..., valueN)) +Insert values into a table. Arguments 2 and 3 may be **tuples** or **list**. + +```python +>>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) +>>> response.json() # result prettyfied just for the example +{ + "query": { + "count": 1, + "created": "2015-05-14T13:25:56Z", + "lang": "en-US", + "results": { + "inserted": { + "execute": "store://KkkC5xDw4v32IcWWSQ4YRe", + "select": "store://Zc5LHXcmYM7XBfSbo9tzFL", + "update": "store://Rqb5fbQyDvrfHJiClWnZ6q" + } + } + } +} +``` + +####update(table,[field1, ..., fieldN],[value1, ..., ...valueN]).where(filters, ...) +Update fields values. This method __is always followed by ***where()***__. Arguments 2 and 3 may be **tuples** or **list**. + +```python +>>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) +>>> response.json() # result prettyfied just for the example +{ + "query": { + "count": 1, + "created": "2015-05-14T13:32:52Z", + "lang": "en-US", + "results": { + "success": "Updated store://KkkC5xDw4v32IcWWSQ4YRe" + } + } +} +``` + +####delete(table).where(filters, ...) +Delete records +```python +>>> response = self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) +>>> response.json() # result prettyfied just for the example +{ + "query": { + "count": 1, + "created": "2015-05-14T13:38:28Z", + "lang": "en-US", + "results": { + "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" + } + } +} + +``` + Using OAuth to fetch protected resources ========================================= diff --git a/docs/myql.md b/docs/myql.md index 3c82400..bcf01ed 100644 --- a/docs/myql.md +++ b/docs/myql.md @@ -64,6 +64,7 @@ Get **items** from **table**. #### *MQYL.select(table=None, items=[], limit=None)* +This method is always followed by a **where**. It doesn't return a response if called alone. * ***table*** : Table name * ***items*** : Element/columns to get from the table @@ -73,6 +74,35 @@ Get **items** from **table**. >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) ``` +#### *MYQL.insert(table,[field1, field2, ..., fieldN],[value1, value2, ..., valueN])* +* ***table***: Table name +* ***fields***: List or Tuple of fields +* ***values***: List or Tuple of values + +```python +>>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) +``` + +#### *MYQL.update(table,[field1, field2, ..., fieldN],[value1, value2, ..., valueN])* +This method is always followed by a **where**. It doesn't return a response if called alone. + +* ***table***: Table name +* ***fields***: List or Tuple of fields to update +* ***values***: List or Tuple of new values + +```python +>>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=',"store://Rqb5fbQyDvrfHJiClWnZ6q"]) +``` + +#### *MYQL.delete(table)* +This method is always followed by a **where**. It doesn t return a response if called alone. + +* ***table***: Table name + +```python +>>> response = self.yql.delete('yql.storage').where(['name','=',"store://Rqb5fbQyDvrfHJiClWnZ6q"]) +``` + #### *MQYL.where(\*args)* * ***\*args*** : List of conditions From 72589b3e6c5cbc226a1132dd3f0d11fe97ee6131 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 14 May 2015 16:30:03 +0200 Subject: [PATCH 318/582] #1: Updating tests doc --- tests/README.md | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/tests/README.md b/tests/README.md index 0f81fa2..0741915 100755 --- a/tests/README.md +++ b/tests/README.md @@ -1,30 +1,10 @@ How to test the Library (if cloned the rep) ======================= -Get to the root directory of the module - -* Config file test - ```shell -python -m unittest tests.LokingyqlTestCase.test_config -. ----------------------------------------------------------------------- -Ran 1 test in 0.000s - -OK -``` - -* OAuth Config : Don't forget to put your credentials in the test_config.py - -```shell -(lokingYQL)josue@LokingMac:~/Dropbox/Workspace/lokingYQL$ python -m unittest tests.LokingyqlTestCase -Please input the verifier : pc25wf -. ----------------------------------------------------------------------- -Ran 1 test in 18.488s - -OK -(lokingYQL)josue@LokingMac:~/Dropbox/Workspace/lokingYQL$ +$ ./run_tests.sh # To run a specific test of a TestCase +$ ./run_tests.sh # To run all tests of a testcase +$ ./run_tests.sh # To run all tests ``` From d1c64305ea3b11b95248f0678a78bcb4cfcfab33 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 14 May 2015 16:30:25 +0200 Subject: [PATCH 319/582] #1: updating contribution documentation --- docs/contrib.md | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/contrib.md b/docs/contrib.md index c22556a..6dd3676 100644 --- a/docs/contrib.md +++ b/docs/contrib.md @@ -1,2 +1,34 @@ -##Contrib +##How to contribute +It's easy to contribute to ***MYQL***. +1. Fork the repository +2. Develop your patches/fixes/features +3. Create a merge request +That's all + +###Tips + +If you want to add a new feature to the library, you better put it in ***myql/contrib/my_awesome_feature/*** + +Let's say i want to add a **weather** module. + +```shell +$ mkdir myql/contrib/weather/ +$ vim myql/contrib/__init__.py +``` + +```python +... +import weather +``` + +```shell +$ vim myql/contrib/weather/__init__.py +from weather import my_stuff +$ vim myql/contrib/weather/weather.py +``` + +```python +from myql.myql import MYQL # If ever you want to use the module +# Your code +``` From 37872b819f2e1df6bbb798871cb57a8b8bbc0704 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 15 May 2015 11:09:37 +0200 Subject: [PATCH 320/582] Cleaning the auth house --- myql/contrib/auth/__init__.py | 1 - myql/contrib/auth/auth.py | 142 ---------------------------------- 2 files changed, 143 deletions(-) delete mode 100644 myql/contrib/auth/auth.py diff --git a/myql/contrib/auth/__init__.py b/myql/contrib/auth/__init__.py index b653833..1a7e3eb 100755 --- a/myql/contrib/auth/__init__.py +++ b/myql/contrib/auth/__init__.py @@ -1,2 +1 @@ -from auth import OAuth from yoauth import YOAuth diff --git a/myql/contrib/auth/auth.py b/myql/contrib/auth/auth.py deleted file mode 100644 index af33348..0000000 --- a/myql/contrib/auth/auth.py +++ /dev/null @@ -1,142 +0,0 @@ -import pdb -import json -import time -import logging -import requests -from requests_oauthlib import OAuth1 -from urlparse import parse_qs -import webbrowser - -BASE_URL = "http://query.yahooapis.com/v1/yql" -REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token" -ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token" -AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth?oauth_token=" -CALLBACK_URI = 'oob' - -logging.basicConfig(level=logging.DEBUG, format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") -logging.getLogger(__name__) - -class OAuth(object): - """ - """ - def __init__(self, consumer_key, consumer_secret, **kwargs): - """ - """ - if kwargs.get('from_file'): - from_file = kwargs.get('from_file') - json_data = self.json_get_data(from_file) - vars(self).update(json_data) - else: - self.consumer_key = consumer_key - self.consumer_secret = consumer_secret - vars(self).update(kwargs) - - if not vars(self).get('access_token') and not vars(self).get('access_token_secret'): - self.get_request_token() - self.verifier = self.get_user_authorization() - self.get_access_token() - # if not vars(self).get('request_token') and not vars(self).get('request_token_secret'): - # self.get_request_token() - # self.verifier = self.get_user_authorization() - # self.get_access_token() - # elif not vars(self).get('verifier'): - # self.verifier = self.get_user_authorization() - # self.get_access_token() - # else: - # self.get_access_token() - #else: - self.oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.access_token, resource_owner_secret=self.access_token_secret) - - json_data.update({ - #'request_token': self.request_token, - #'request_token_secret': self.request_token_secret, - #'verifier': self.verifier, - 'access_token': self.access_token, - 'access_token_secret': self.access_token_secret, - 'session_handle': self.session_handle, - 'token_time': time.time() - }) - - self.json_wirte_data(json_data, from_file) - - def isValid(self): - """Check if the token hasn't expired - """ - elapsed_time = time.time() - vars(self).get('token_time',0) - if elapsed_time > 3000 and elapsed_time < 3600: - self.refresh_token() - logging.debug("Token Refreshed") - True - elif elapsed_time > 3600: - return False - return True - - def refresh_token(self): - """Refresh access token - """ - #oauth = OAuth1(self.consumer_key, resource_owner_key=self.access_token,resource_owner_secret=self.access_token_secret) - oauth = OAuth1(resource_owner_key=self.access_token,resource_owner_secret=self.access_token_secret) - response = requests.post(ACCESS_TOKEN_URL, headers={'oauth_session_handle': self.session_handle}, auth=oauth) - pdb.set_trace() - tokens = self.fetch_tokens(response.content) - return tokens - - def fetch_tokens(self, content): - """Parse content to fetch request/access token/token-secret - """ - stuff = parse_qs(content) - stuff = {k:v[0] for (k,v) in stuff.items()} - return stuff - - def json_get_data(self, filename): - """Returns content of a json file - """ - with open(filename) as fp: - json_data = json.load(fp) - - return json_data - - def json_wirte_data(self, json_data, filename): - """Write data into a json file - """ - with open(filename, 'w') as fp: - json.dump(json_data, fp, indent=4, encoding= 'utf-8', sort_keys=True) - return True - - return False - - def get_request_token(self,): - """Get request token - """ - oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, callback_uri=CALLBACK_URI) - response = requests.post(url=REQUEST_TOKEN_URL, auth=oauth) - tokens = self.fetch_tokens(response.content) - logging.debug(tokens) - self.request_token, self.request_token_secret = tokens.get('oauth_token'), tokens.get('oauth_token_secret') - logging.debug("{0}, {1}".format(self.request_token, self.request_token_secret)) - return tokens - - def get_user_authorization(self,): - """Get authorization - """ - authorization_url = AUTHORIZE_TOKEN_URL+self.request_token - logging.debug(authorization_url) - webbrowser.open(authorization_url) - verifier = raw_input("Please input a verifier: ") - logging.debug(verifier) - return verifier - - def get_access_token(self): - """Get access token - """ - oauth = OAuth1(self.consumer_key, client_secret=self.consumer_secret, resource_owner_key=self.request_token, resource_owner_secret=self.request_token_secret,verifier=self.verifier) - response = requests.post(url=ACCESS_TOKEN_URL, auth=oauth) - tokens = self.fetch_tokens(response.content) - logging.debug(tokens) - self.access_token, self.access_token_secret, self.session_handle = tokens.get('oauth_token'), tokens.get('oauth_token_secret'), tokens.get('oauth_session_handle') - logging.debug("{0} {1}".format(self.access_token, self.access_token_secret)) - return tokens - -if '__main__' == __name__: - auth = OAuth(None, None, from_file='credentials.json') - auth.refresh_token() From 797d1864f9b1065d7a598f3366d0702c9c4fd44b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 15 May 2015 14:15:15 +0200 Subject: [PATCH 321/582] #1 : Release Notes added on #64 --- README.md | 2 +- myql/contrib/auth/yoauth.py | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index efcf442..45242b0 100755 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ v 1.2.2 ( development ) * Import of StockParser from Gurchet Rai https://github.com/gurch101/StockScraper OK [#68](https://github.com/josuebrunel/myql/issues/68) * Insert, Update, Delete methods added [#67](https://github.com/josuebrunel/myql/issues/67) * Dummy *try/except* removed from main module - +* Fixed **Invalid OAuth Signature** when using a refreshed token [#64](https://github.com/josuebrunel/myql/issues/64) v 1.2.1 ------ diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index 638933c..e4c8cd4 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -31,6 +31,7 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): from_file : file containing the credentials """ if kwargs.get('from_file'): + logging.debug("Checking ") self.from_file = kwargs.get('from_file') json_data = self.json_get_data(self.from_file) vars(self).update(json_data) @@ -53,8 +54,6 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): if vars(self).get('access_token') and vars(self).get('access_token_secret') and vars(self).get('session_handle'): if not self.token_is_valid(): self.session = self.refresh_token() - else: - self.session = self.oauth.get_session((self.access_token, self.access_token_secret)) else: # Fetching request token/token_secret request_token, request_token_secret = self.oauth.get_request_token(params={'oauth_callback': CALLBACK_URI}) @@ -74,7 +73,7 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): self.access_token_secret = parsed_acess['oauth_token_secret'] self.session_handle = parsed_acess['oauth_session_handle'] - self.session = self.oauth.get_session((self.access_token, self.access_token_secret)) + self.session = self.oauth.get_session((self.access_token, self.access_token_secret)) json_data.update({ 'access_token' : self.access_token, @@ -108,7 +107,7 @@ def refresh_token(self,): """ logging.debug("REFRESHING TOKEN") self.token_time = time.time() - self.acess_token, self.access_token_secret = self.oauth.get_access_token(self.access_token, self.access_token_secret, params={"oauth_session_handle": self.session_handle}) + self.access_token, self.access_token_secret = self.oauth.get_access_token(self.access_token, self.access_token_secret, params={"oauth_session_handle": self.session_handle}) session = self.oauth.get_session((self.access_token, self.access_token_secret)) @@ -119,15 +118,10 @@ def token_is_valid(self,): """ elapsed_time = time.time() - self.token_time logging.debug("ELAPSED TIME : {0}".format(elapsed_time)) - if elapsed_time > 3600: + if elapsed_time > 3300: # 5 minutes before it expires logging.debug("TOKEN HAS EXPIRED") return False logging.debug("TOKEN IS STILL VALID") return True - - -if '__main__' == __name__: - auth = YOAuth(None, None, from_file='credentials.json') - From 5a7d58b13eb9380e9803de0a28f12753d6c52bc4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 15 May 2015 14:25:23 +0200 Subject: [PATCH 322/582] #1: License changed from GNU to MIT --- LICENSE | 351 +++---------------------------------------------------- setup.py | 4 +- 2 files changed, 19 insertions(+), 336 deletions(-) diff --git a/LICENSE b/LICENSE index d7f1051..c304a72 100755 --- a/LICENSE +++ b/LICENSE @@ -1,339 +1,22 @@ -GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 +The MIT License (MIT) - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. +Copyright (c) 2015 josuebrunel - Preamble +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - {description} - Copyright (C) {year} {fullname} - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - {signature of Ty Coon}, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/setup.py b/setup.py index c2026b5..9570705 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import os from setuptools import setup, find_packages -__version__ = "1.2.1" +__version__ = "1.2.2" #requirements.txt with open('requirements.txt') as f: @@ -29,6 +29,6 @@ def read(fname): 'License :: OSI Approved :: MIT License' ], platforms=['Any'], - license='GNU', + license='MIT', install_requires = required ) From 5c9a4db75b73c7db15e2a682fb27ae00e00db988 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 15 May 2015 15:00:26 +0200 Subject: [PATCH 323/582] fix #64 : Invalid OAuth Signature OK --- myql/contrib/auth/yoauth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index e4c8cd4..7134eed 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -118,7 +118,7 @@ def token_is_valid(self,): """ elapsed_time = time.time() - self.token_time logging.debug("ELAPSED TIME : {0}".format(elapsed_time)) - if elapsed_time > 3300: # 5 minutes before it expires + if elapsed_time > 3540: # 1 minute before it expires logging.debug("TOKEN HAS EXPIRED") return False From fd9448eaa185ac9483522e62af477304767cc322 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 17 May 2015 06:52:20 +0200 Subject: [PATCH 324/582] test_use added --- tests/tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 82da9c2..dba0075 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -45,6 +45,16 @@ def setUp(self,): def tearDown(self): pass + def test_use(self): + self.yql.use('http://www.josuebrunel.org/users.xml',name='users') + response = self.yql.rawQuery('select * from users') + self.yql.yql_table_url = None + try: + logging.debug(pretty_json(response.content)) + except Exception,e: + logging.error(e) + self.assertEquals(response.status_code,200) + def test_raw_query(self,): response = self.yql.rawQuery('select name, woeid from geo.states where place="Congo"') try: From ae3b92ccedf8ceff38fbfe1efae77b76092cc0c9 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 17 May 2015 06:53:28 +0200 Subject: [PATCH 325/582] fix #76: MYQL.use not misused anymore --- myql/myql.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 779a9db..4114bdc 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -50,6 +50,10 @@ def payloadBuilder(self, query, format='json'): '''Build the payload''' if self.community : query = self.community_data + query # access to community data tables + + if self.use : + query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query + logger.debug(query) payload = { 'q' : query, @@ -132,12 +136,13 @@ def buildResponse(self, response): ##################################################### ##USE - def use(self, url): + def use(self, url, name='mytable'): '''Changes the data provider >>> yql.use('http://myserver.com/mytables.xml') ''' - self.url = url - return self.url + self.yql_table_url = url + self.yql_table_name = name + return {'table url': url, 'table name': name} ##DESC def desc(self, table=None): From 851ff85e8a659df3608012a023076562b663e200 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 17 May 2015 07:05:08 +0200 Subject: [PATCH 326/582] fix #76 : better fix --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 4114bdc..7ee47e7 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -51,7 +51,7 @@ def payloadBuilder(self, query, format='json'): if self.community : query = self.community_data + query # access to community data tables - if self.use : + if vars(self).get('yql_table_url') : # Attribute only defined when MYQL.use has been called before query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query logger.debug(query) From de4046bb1cfc2d73360e4bff203e249fbcea9418 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 18 May 2015 13:25:33 +0200 Subject: [PATCH 327/582] #1: improving docs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 45242b0..12c7e6a 100755 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ v 1.2.2 ( development ) * Insert, Update, Delete methods added [#67](https://github.com/josuebrunel/myql/issues/67) * Dummy *try/except* removed from main module * Fixed **Invalid OAuth Signature** when using a refreshed token [#64](https://github.com/josuebrunel/myql/issues/64) +* Fixed misused of ***MYQL.use(...)*** [#76](https://github.com/josuebrunel/myql/issues/76) v 1.2.1 ------ From 3ad6fdede3337b258217e7cca60568ab4d34b060 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 18 May 2015 13:31:44 +0200 Subject: [PATCH 328/582] #1 : updating use method documentation --- README.md | 4 +-- README.rst | 88 ++++++++++++++++++++++++++++++++++++++++++++++++--- docs/index.md | 4 +-- docs/myql.md | 4 +-- 4 files changed, 90 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 12c7e6a..748aa76 100755 --- a/README.md +++ b/README.md @@ -113,11 +113,11 @@ u'\n>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) + >>> response.json() # result prettyfied just for the example + { + "query": { + "count": 1, + "created": "2015-05-14T13:32:52Z", + "lang": "en-US", + "results": { + "success": "Updated store://KkkC5xDw4v32IcWWSQ4YRe" + } + } + } + +delete(table).where(filters, ...) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Delete records \`\`\`python >>> response = +self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) +>>> response.json() # result prettyfied just for the example { "query": +{ "count": 1, "created": "2015-05-14T13:38:28Z", "lang": "en-US", +"results": { "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" } } } + +\`\`\` + Using OAuth to fetch protected resources ======================================== @@ -193,7 +270,10 @@ Using OAuth to fetch protected resources Status| image:: https://travis-ci.org/josuebrunel/myql.svg?branch=master .. |Documentation Status| image:: https://readthedocs.org/projects/myql/badge/?version=latest -.. |PyPI version| image:: https://badge.fury.io/py/myql.svg +.. |Latest Version| image:: https://pypip.in/version/myql/badge.svg +.. |Downloads| image:: https://pypip.in/download/myql/badge.svg +.. |Status| image:: https://pypip.in/py_versions/myql/badge.svg +.. |image5| image:: https://pypip.in/implementation/myql/badge.svg .. |Join the chat at https://gitter.im/josuebrunel/myql| image:: https://badges.gitter.im/Join%20Chat.svg .. |Code diff --git a/docs/index.md b/docs/index.md index c38b56d..88acbd3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -77,11 +77,11 @@ u'\n Date: Tue, 19 May 2015 10:04:20 +0200 Subject: [PATCH 330/582] #71: Turning exception into Python3 exceptions --- tests/tests.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index dba0075..2a10b25 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -51,7 +51,7 @@ def test_use(self): self.yql.yql_table_url = None try: logging.debug(pretty_json(response.content)) - except Exception,e: + except (Exception,) as e: logging.error(e) self.assertEquals(response.status_code,200) @@ -59,7 +59,7 @@ def test_raw_query(self,): response = self.yql.rawQuery('select name, woeid from geo.states where place="Congo"') try: logging.debug(pretty_json(response.content)) - except Exception,e: + except (Exception,) as e: logging.error(e) self.assertEquals(response.status_code,200) @@ -67,7 +67,7 @@ def test_get(self,): response = self.yql.get('geo.countries', ['name', 'woeid'], 1) try: logging.debug(pretty_json(response.content)) - except Exception,e: + except (Exception,) as e: logging.error(e) self.assertEquals(response.status_code,200) @@ -75,7 +75,7 @@ def test_select(self,): response = self.yql.select('geo.countries', ['name', 'code', 'woeid']).where(['name', '=', 'Canada']) try: logging.debug(pretty_json(response.content)) - except Exception,e: + except (Exception,) as e: logging.error(e) self.assertEquals(response.status_code,200) @@ -83,7 +83,7 @@ def test_select_in(self,): response = self.yql.select('yahoo.finance.quotes').where(['symbol','in',("YHOO","AAPL","GOOG")]) try: logging.debug(pretty_json(response.content)) - except Exception,e: + except (Exception,) as e: logging.error(e) self.assertEquals(response.status_code,200) @@ -94,7 +94,7 @@ def test_insert(self,): data = response.json()['query']['results']['inserted'] logging.debug(data) json_write_data(data,'yql_storage.json') - except Exception,e: + except (Exception,) as e: logging.error(response.content) logging.error(e) @@ -105,7 +105,7 @@ def test_check_insert(self,): response = self.yql.select('yql.storage').where(['name','=',json_data['select']]) try: logging.debug(pretty_json(response.content)) - except Exception,e: + except (Exception,) as e: logging.error(response.content) logging.error(e) @@ -116,7 +116,7 @@ def test_update(self,): response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=',json_data['update']]) try: logging.debug(pretty_json(response.content)) - except Exception,e: + except (Exception,) as e: logging.error(response.content) logging.error(e) @@ -127,7 +127,7 @@ def test_delete(self,): response = self.yql.delete('yql.storage').where(['name','=',json_data['update']]) try: logging.debug(pretty_json(response.content)) - except Exception,e: + except (Exception,) as e: logging.error(response.content) logging.error(e) From 29f9a38239b83090086cd2cc1ad9736fefcedab3 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 19 May 2015 10:09:21 +0200 Subject: [PATCH 331/582] #71: running toward python3 --- myql/contrib/table/base.py | 10 +++++----- myql/contrib/table/table.py | 2 +- myql/myql.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index e9425a1..4d1ea01 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -18,7 +18,7 @@ def removeElement(self, elt, tagName=None): try: root.remove(t_elt) return True - except Exception,e: + except (Exception,) as e: print(e) return False @@ -48,7 +48,7 @@ def removeFunction(self): try: root.remove(t_execute) return True - except Exception,e: + except (Exception,) as e: print(e) return False @@ -157,7 +157,7 @@ def removeInput(self, key_id, input_type='key'): try: t_inputs.remove(key[0]) return True - except Exception,e: + except (Exception,) as e: print(e) return False @@ -172,7 +172,7 @@ def addPaging(self,paging): try: root.append(paging.etree) return True - except Exception,e: + except (Exception,) as e: print(e) return False @@ -186,7 +186,7 @@ def removePaging(self,): try: root.remove(t_paging) return True - except Exception,e: + except (Exception,) as e: print(e) return False diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index 6f97510..2bfd2b8 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -86,7 +86,7 @@ def save(self, name=None, path=None): try: self._create_table_xml_file(self.etree, name) - except Exception,e: + except (Exception,) as e: print(e) return False diff --git a/myql/myql.py b/myql/myql.py index 0688211..9f651d0 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -127,7 +127,7 @@ def buildResponse(self, response): 'num_result': r['query']['count'] , 'result': result } - except Exception, e: + except (Exception,) as e: print(e) return response.content return response @@ -195,7 +195,7 @@ def select(self, table=None, items=[], limit=''): self._query = "select {1} from {0} ".format(self.table, ','.join(items)) try: #Checking wether a limit is set or not self._limit = limit - except Exception, e: + except (Exception,) as e: pass return self From 382f3b3886788fbbcb54310ccb3eeceba5b704b4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 10:02:20 +0200 Subject: [PATCH 332/582] #68 : updating stockscraper doc --- docs/myql.md | 2 +- docs/stockscraper.md | 87 ++++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 docs/stockscraper.md diff --git a/docs/myql.md b/docs/myql.md index a3c2d1c..3b9593d 100644 --- a/docs/myql.md +++ b/docs/myql.md @@ -3,7 +3,7 @@ MYQL ### **Definition** -#### *MYQL(community=True, fromat='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None)* +#### *MYQL(community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None)* ### **Methods** diff --git a/docs/stockscraper.md b/docs/stockscraper.md new file mode 100644 index 0000000..39211bf --- /dev/null +++ b/docs/stockscraper.md @@ -0,0 +1,87 @@ +StockScraper +============ + +***[StockScraper](https://github.com/gurch101/StockScraper)*** is a module written by **[Gurchet Rai](https://github.com/gurch101/)** and has just been imported into *mYQL*. + +Full [Documentation](http://www.gurchet-rai.net/dev/yahoo-finance-yql) + +### **Definition** + +#### *StockRetriever(format='json', debug=False, oauth=None)* + +* ***format*** : xml or json +* ***debug*** : True or False +* ***oauth*** : YOAuth object + +```python +from myql.contrib.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +``` + +### **Methods** + +#### *StockRetriever.get_current_info(symbolList, columns=None)* + +* ***symbolList*** : List of symbol to retrieve +* ***columns*** : List of column to fetch + +```python +from myql.contrib.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_current_info(["YHOO","AAPL","GOOG"]) +``` + +#### *StockRetriever.get_news_feed(symbol)* + +* ***symbol*** : Symbol news to retrieve + +```python +from myql.contrib.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_news_feed('YHOO') +``` + +#### *StockRetriever.get_historical_info(symbol)* + +* ***symbol*** : Symbol news to retrieve + +```python +from myql.contrib.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_historical_info('YHOO') +``` + +#### *StockRetriever.get_options_info(symbol, items=[], expiration=None)* + +* ***symbol*** : Symbol news to retrieve +* ***items*** : list of attributes to retrieve +* ***expiration*** : Date of expiration (type : str) + +```python +from myql.contrib.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_options_info('YHOO') +``` + +#### *StockRetriever.get_index_summary(symbol, items=[])* + +* ***symbol*** : Symbol news to retrieve +* ***items*** : list of attributes to retrieve + +```python +from myql.contrib.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_index_summary('GOOG',('Volume','Change')) +``` + +#### *StockRetriever.get_industry_index(index_id,items=[])* + +* ***index_id*** : index id +* ***items*** : list of attributes to retrieve + +```python +from myql.contrib.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_industry_index(112) +``` + diff --git a/mkdocs.yml b/mkdocs.yml index e073cae..f985a4d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,6 +7,7 @@ google_analytics: ['UA-32441224-4', 'readthedocs.org'] pages: - [index.md, Home] - [myql.md, MYQL] +- [stockscraper.md, StockScraper] - [oauth.md, YOAuth] - [table.md, Open Table] - [contrib.md, Contribute] From 28731fb8355b23e23966e88f2e2180d3ac77ef31 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Wed, 20 May 2015 11:27:14 +0200 Subject: [PATCH 333/582] improving rst --- README.rst | 245 ++++++++--------------------------------------------- 1 file changed, 37 insertions(+), 208 deletions(-) diff --git a/README.rst b/README.rst index e08f0aa..a499054 100644 --- a/README.rst +++ b/README.rst @@ -1,15 +1,12 @@ MYQL ==== -`|Build Status| `_ -`|Documentation Status| `_ `|Latest -Version| `_ -`|Downloads| `_ -`|Status| `_ -`|image5| `_ `|Join the chat at -https://gitter.im/josuebrunel/myql| `_ -`|Code -Issues| `_ +| |Build Status| |Documentation Status| +| |Latest Version| +| |Downloads| +| |Status| +| |Status| +| |Join the chat at https://gitter.im/josuebrunel/myql| |Code Issues| MYQL is a Python wrapper of the Yahoo Query Language. @@ -33,15 +30,11 @@ v 1.2.2 ( development ) - Fixed issue when passing an empty list/tuple (**[]/()**) in a **where** clause - Import of StockParser from Gurchet Rai - https://github.com/gurch101/StockScraper OK - `#68 `_ -- Insert, Update, Delete methods added - `#67 `_ + https://github.com/gurch101/StockScraper OK `#68`_ +- Insert, Update, Delete methods added `#67`_ - Dummy *try/except* removed from main module -- Fixed **Invalid OAuth Signature** when using a refreshed token - `#64 `_ -- Fixed misused of ***MYQL.use(...)*** - `#76 `_ +- Fixed **Invalid OAuth Signature** when using a refreshed token `#64`_ +- Fixed misused of ***MYQL.use(…)*** `#76`_ v 1.2.1 ------- @@ -63,21 +56,20 @@ v 0.5.6 - change data source - filter data - fix handling of default and on the fly response format -- fix limit on ***select(...).where(...)*** when no limit value is - passed -- fix limit on ***get(...)*** +- fix limit on ***select(…).where(…)*** when no limit value is passed +- fix limit on ***get(…)*** installation ============ -:: +.. code:: shell $ pip install myql how to use ========== -:: +.. code:: python >>> import myql >>> yql = myql.MYQL() @@ -86,195 +78,32 @@ how to use access to community tables ^^^^^^^^^^^^^^^^^^^^^^^^^^ -:: +\`\`\`python - >>> yql = myql.MYQL() - >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') - >>> rep.json() - {u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} - >>> yql.community= True # Setting up access to community - >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') - >>> rep.json() - {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} - >>> - -***OR*** - -:: - - >>> import myql - >>> yql = myql.MYQL(community=True) - >>> # do your magic - -changing response format (xml or json) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The response format is by default ***json***. - -:: - - >>> import myql - >>> yql = myql.MYQL(format='xml', community=True) - >>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') - >>> rep.text - u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' - >>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') - >>> rep.json() - {u'query': {u'count': 11, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'55998384', u'name': u'Cuvette-Ouest Department'}, {u'woeid': u'2344968', u'name': u'Cuvette Department'}, {u'woeid': u'2344973', u'name': u'Plateaux District'}, {u'woeid': u'2344974', u'name': u'Sangha'}, {u'woeid': u'2344970', u'name': u'Lekoumou'}, {u'woeid': u'2344975', u'name': u'Pool Department'}, {u'woeid': u'2344971', u'name': u'Likouala Department'}, {u'woeid': u'2344972', u'name': u'Niari Department'}, {u'woeid': u'2344976', u'name': u'Brazzaville'}, {u'woeid': u'2344967', u'name': u'Bouenza Department'}, {u'woeid': u'2344969', u'name': u'Kouilou'}]}, u'created': u'2014-08-27T04:52:38Z'}} - >>> - -Methods -------- - -use(data\_provider\_url) -^^^^^^^^^^^^^^^^^^^^^^^^ - -Changes the data provider - -:: - - >>> yql.use('http://myserver.com/mytables.xml') - -desc(tablename) -^^^^^^^^^^^^^^^ - -Returns table description - -:: - - >>> response = yql.desc('weather.forecast') - >>> response.json() - {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'request': {u'select': [{u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'location'}, {u'type': u'xs:string', u'name': u'u'}]}, {u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'woeid'}, {u'type': u'xs:string', u'name': u'u'}]}]}, u'security': u'ANY', u'meta': {u'documentationURL': u'http://developer.yahoo.com/weather/', u'sampleQuery': u'select * from weather.forecast where woeid=2502265', u'description': u'Weather forecast table', u'author': u'Yahoo! Inc'}, u'hash': u'aae78b1462a6a8fbc748aec4cf292767', u'name': u'weather.forecast'}}, u'created': u'2014-08-16T19:31:51Z'}} - >>> - -rawQuery(query) -^^^^^^^^^^^^^^^ - -Allows you to directly type your query - -:: - - >>> response = yql.rawQuery("select * from geo.countries where place='North America'") - >>> # deal with the response - -select(table, fields, limit).where(filters, ...) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Select a table i.e *weather.forecast*. If *table* not provided, it will -use the default table. If there's no such thing as a default table, it -will raise a *NoTableSelectedError* - -***NB*** : A simple select doesn't return any data. Use ***GET*** -instead. - -:: - - >>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) - >>> response.json() - {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424775', u'name': u'Canada'}}, u'created': u'2014-08-16T19:04:08Z'}} - >>> ... - >>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', '=', 'Africa']) - >>> rep.json() - {u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T10:52:49Z'}} - >>> - >>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) - >>> rep.json() - {u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T11:22:49Z'}} - >>> - -get(table, fields, limit) -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Same as ***SELECT***, but instead returns data. - -**REMINDER** : Some tables require a **where clause**, therefore -***GET*** won't work on those tables, use *select(...).where(...)* -instead . - -:: - - >>> rep = yql.get('geo.countries', ['name', 'woeid'], 1) - >>> rep.json() - {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} - >>> - -insert(table, (field1, field2, ..., fieldN),(value1, value2, ..., valueN)) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Insert values into a table. Arguments 2 and 3 may be **tuples** or -**list**. - -:: - - >>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) - >>> response.json() # result prettyfied just for the example - { - "query": { - "count": 1, - "created": "2015-05-14T13:25:56Z", - "lang": "en-US", - "results": { - "inserted": { - "execute": "store://KkkC5xDw4v32IcWWSQ4YRe", - "select": "store://Zc5LHXcmYM7XBfSbo9tzFL", - "update": "store://Rqb5fbQyDvrfHJiClWnZ6q" - } - } - } - } - -update(table,[field1, ..., fieldN],[value1, ..., ...valueN]).where(filters, ...) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Update fields values. This method **is always followed by -***where()*****. Arguments 2 and 3 may be **tuples** or **list**. - -:: - - >>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) - >>> response.json() # result prettyfied just for the example - { - "query": { - "count": 1, - "created": "2015-05-14T13:32:52Z", - "lang": "en-US", - "results": { - "success": "Updated store://KkkC5xDw4v32IcWWSQ4YRe" - } - } - } - -delete(table).where(filters, ...) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Delete records \`\`\`python >>> response = -self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) ->>> response.json() # result prettyfied just for the example { "query": -{ "count": 1, "created": "2015-05-14T13:38:28Z", "lang": "en-US", -"results": { "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" } } } - -\`\`\` - -Using OAuth to fetch protected resources -======================================== - -:: + | yql = myql.MYQL() + | rep = yql.rawQuery(‘desc yahoo.finance.quotes’) + | rep.json() + | {u’error’: {u’lang’: u’en-US’, u’description’: u’No + definition found for Table yahoo.finance.quotes’}} - >>> from myql.contrib.auth import YOAuth - >>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. - >>> from myql import MYQL - >>> yql = MYQL(format='xml', oauth=oauth) - >>> response = yql.getGUID('josue_brunel') # Deal with the response +.. _#68: https://github.com/josuebrunel/myql/issues/68 +.. _#67: https://github.com/josuebrunel/myql/issues/67 +.. _#64: https://github.com/josuebrunel/myql/issues/64 +.. _#76: https://github.com/josuebrunel/myql/issues/76 -.. |Build -Status| image:: https://travis-ci.org/josuebrunel/myql.svg?branch=master -.. |Documentation -Status| image:: https://readthedocs.org/projects/myql/badge/?version=latest +.. |Build Status| image:: https://travis-ci.org/josuebrunel/myql.svg?branch=master + :target: https://travis-ci.org/josuebrunel/myql +.. |Documentation Status| image:: https://readthedocs.org/projects/myql/badge/?version=latest + :target: https://myql.readthedocs.org .. |Latest Version| image:: https://pypip.in/version/myql/badge.svg + :target: https://pypi.python.org/pypi/myql/ .. |Downloads| image:: https://pypip.in/download/myql/badge.svg + :target: https://pypi.python.org/pypi/myql .. |Status| image:: https://pypip.in/py_versions/myql/badge.svg -.. |image5| image:: https://pypip.in/implementation/myql/badge.svg -.. |Join the chat at -https://gitter.im/josuebrunel/myql| image:: https://badges.gitter.im/Join%20Chat.svg -.. |Code -Issues| image:: https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg + :target: https://pypi.python.org/pypi/myql +.. |Status| image:: https://pypip.in/implementation/myql/badge.svg + :target: https://pypi.python.org/pypi/myql +.. |Join the chat at https://gitter.im/josuebrunel/myql| image:: https://badges.gitter.im/Join%20Chat.svg + :target: https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +.. |Code Issues| image:: https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg + :target: https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql From 6346feda8b25c7bdb50f222a45e9fbd9cba25b7d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 11:40:02 +0200 Subject: [PATCH 334/582] #1: trying to generate the right rst file --- README.rst | 243 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 206 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index a499054..9cfb9cc 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,15 @@ MYQL ==== -| |Build Status| |Documentation Status| -| |Latest Version| -| |Downloads| -| |Status| -| |Status| -| |Join the chat at https://gitter.im/josuebrunel/myql| |Code Issues| +` `_ `|Documentation +Status| `_ `|Latest +Version| `_ +`|Downloads| `_ +`|Status| `_ +`|image4| `_ `|Join the chat at +https://gitter.im/josuebrunel/myql| `_ +`|Code +Issues| `_ MYQL is a Python wrapper of the Yahoo Query Language. @@ -30,11 +33,15 @@ v 1.2.2 ( development ) - Fixed issue when passing an empty list/tuple (**[]/()**) in a **where** clause - Import of StockParser from Gurchet Rai - https://github.com/gurch101/StockScraper OK `#68`_ -- Insert, Update, Delete methods added `#67`_ + https://github.com/gurch101/StockScraper OK + `#68 `_ +- Insert, Update, Delete methods added + `#67 `_ - Dummy *try/except* removed from main module -- Fixed **Invalid OAuth Signature** when using a refreshed token `#64`_ -- Fixed misused of ***MYQL.use(…)*** `#76`_ +- Fixed **Invalid OAuth Signature** when using a refreshed token + `#64 `_ +- Fixed misused of ***MYQL.use(...)*** + `#76 `_ v 1.2.1 ------- @@ -56,20 +63,21 @@ v 0.5.6 - change data source - filter data - fix handling of default and on the fly response format -- fix limit on ***select(…).where(…)*** when no limit value is passed -- fix limit on ***get(…)*** +- fix limit on ***select(...).where(...)*** when no limit value is + passed +- fix limit on ***get(...)*** installation ============ -.. code:: shell +:: $ pip install myql how to use ========== -.. code:: python +:: >>> import myql >>> yql = myql.MYQL() @@ -78,32 +86,193 @@ how to use access to community tables ^^^^^^^^^^^^^^^^^^^^^^^^^^ -\`\`\`python +:: - | yql = myql.MYQL() - | rep = yql.rawQuery(‘desc yahoo.finance.quotes’) - | rep.json() - | {u’error’: {u’lang’: u’en-US’, u’description’: u’No - definition found for Table yahoo.finance.quotes’}} + >>> yql = myql.MYQL() + >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') + >>> rep.json() + {u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} + >>> yql.community= True # Setting up access to community + >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') + >>> rep.json() + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} + >>> + +***OR*** + +:: + + >>> import myql + >>> yql = myql.MYQL(community=True) + >>> # do your magic + +changing response format (xml or json) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The response format is by default ***json***. + +:: + + >>> import myql + >>> yql = myql.MYQL(format='xml', community=True) + >>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') + >>> rep.text + u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' + >>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') + >>> rep.json() + {u'query': {u'count': 11, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'55998384', u'name': u'Cuvette-Ouest Department'}, {u'woeid': u'2344968', u'name': u'Cuvette Department'}, {u'woeid': u'2344973', u'name': u'Plateaux District'}, {u'woeid': u'2344974', u'name': u'Sangha'}, {u'woeid': u'2344970', u'name': u'Lekoumou'}, {u'woeid': u'2344975', u'name': u'Pool Department'}, {u'woeid': u'2344971', u'name': u'Likouala Department'}, {u'woeid': u'2344972', u'name': u'Niari Department'}, {u'woeid': u'2344976', u'name': u'Brazzaville'}, {u'woeid': u'2344967', u'name': u'Bouenza Department'}, {u'woeid': u'2344969', u'name': u'Kouilou'}]}, u'created': u'2014-08-27T04:52:38Z'}} + >>> + +Methods +------- + +use(yql\_table\_url,name=yql\_table\_name ) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Changes the data provider + +:: + + >>> yql.use('http://www.josuebrunel.org//users.xml', name='myusers') + +desc(tablename) +^^^^^^^^^^^^^^^ + +Returns table description + +:: + + >>> response = yql.desc('weather.forecast') + >>> response.json() + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'request': {u'select': [{u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'location'}, {u'type': u'xs:string', u'name': u'u'}]}, {u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'woeid'}, {u'type': u'xs:string', u'name': u'u'}]}]}, u'security': u'ANY', u'meta': {u'documentationURL': u'http://developer.yahoo.com/weather/', u'sampleQuery': u'select * from weather.forecast where woeid=2502265', u'description': u'Weather forecast table', u'author': u'Yahoo! Inc'}, u'hash': u'aae78b1462a6a8fbc748aec4cf292767', u'name': u'weather.forecast'}}, u'created': u'2014-08-16T19:31:51Z'}} + >>> + +rawQuery(query) +^^^^^^^^^^^^^^^ + +Allows you to directly type your query + +:: + + >>> response = yql.rawQuery("select * from geo.countries where place='North America'") + >>> # deal with the response + +select(table, fields, limit).where(filters, ...) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Select a table i.e *weather.forecast*. If *table* not provided, it will +use the default table. If there's no such thing as a default table, it +will raise a *NoTableSelectedError* + +***NB*** : A simple select doesn't return any data. Use ***GET*** +instead. + +:: + + >>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) + >>> response.json() + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424775', u'name': u'Canada'}}, u'created': u'2014-08-16T19:04:08Z'}} + >>> ... + >>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', '=', 'Africa']) + >>> rep.json() + {u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T10:52:49Z'}} + >>> + >>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) + >>> rep.json() + {u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T11:22:49Z'}} + >>> + +get(table, fields, limit) +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Same as ***SELECT***, but instead returns data. + +**REMINDER** : Some tables require a **where clause**, therefore +***GET*** won't work on those tables, use *select(...).where(...)* +instead . + +:: + + >>> rep = yql.get('geo.countries', ['name', 'woeid'], 1) + >>> rep.json() + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} + >>> + +insert(table, (field1, field2, ..., fieldN),(value1, value2, ..., valueN)) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Insert values into a table. Arguments 2 and 3 may be **tuples** or +**list**. + +:: + + >>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) + >>> response.json() # result prettyfied just for the example + { + "query": { + "count": 1, + "created": "2015-05-14T13:25:56Z", + "lang": "en-US", + "results": { + "inserted": { + "execute": "store://KkkC5xDw4v32IcWWSQ4YRe", + "select": "store://Zc5LHXcmYM7XBfSbo9tzFL", + "update": "store://Rqb5fbQyDvrfHJiClWnZ6q" + } + } + } + } + +update(table,[field1, ..., fieldN],[value1, ..., ...valueN]).where(filters, ...) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Update fields values. This method **is always followed by +***where()*****. Arguments 2 and 3 may be **tuples** or **list**. + +:: + + >>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) + >>> response.json() # result prettyfied just for the example + { + "query": { + "count": 1, + "created": "2015-05-14T13:32:52Z", + "lang": "en-US", + "results": { + "success": "Updated store://KkkC5xDw4v32IcWWSQ4YRe" + } + } + } + +delete(table).where(filters, ...) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Delete records \`\`\`python >>> response = +self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) +>>> response.json() # result prettyfied just for the example { "query": +{ "count": 1, "created": "2015-05-14T13:38:28Z", "lang": "en-US", +"results": { "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" } } } + +\`\`\` + +Using OAuth to fetch protected resources +======================================== + +:: -.. _#68: https://github.com/josuebrunel/myql/issues/68 -.. _#67: https://github.com/josuebrunel/myql/issues/67 -.. _#64: https://github.com/josuebrunel/myql/issues/64 -.. _#76: https://github.com/josuebrunel/myql/issues/76 + >>> from myql.contrib.auth import YOAuth + >>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. + >>> from myql import MYQL + >>> yql = MYQL(format='xml', oauth=oauth) + >>> response = yql.getGUID('josue_brunel') # Deal with the response -.. |Build Status| image:: https://travis-ci.org/josuebrunel/myql.svg?branch=master - :target: https://travis-ci.org/josuebrunel/myql -.. |Documentation Status| image:: https://readthedocs.org/projects/myql/badge/?version=latest - :target: https://myql.readthedocs.org +.. |Documentation +Status| image:: https://readthedocs.org/projects/myql/badge/?version=latest .. |Latest Version| image:: https://pypip.in/version/myql/badge.svg - :target: https://pypi.python.org/pypi/myql/ .. |Downloads| image:: https://pypip.in/download/myql/badge.svg - :target: https://pypi.python.org/pypi/myql .. |Status| image:: https://pypip.in/py_versions/myql/badge.svg - :target: https://pypi.python.org/pypi/myql -.. |Status| image:: https://pypip.in/implementation/myql/badge.svg - :target: https://pypi.python.org/pypi/myql -.. |Join the chat at https://gitter.im/josuebrunel/myql| image:: https://badges.gitter.im/Join%20Chat.svg - :target: https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -.. |Code Issues| image:: https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg - :target: https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql +.. |image4| image:: https://pypip.in/implementation/myql/badge.svg +.. |Join the chat at +https://gitter.im/josuebrunel/myql| image:: https://badges.gitter.im/Join%20Chat.svg +.. |Code +Issues| image:: https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg From a509f3d67a86252c6e355171218de48b35e78d0e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 11:44:09 +0200 Subject: [PATCH 335/582] #1 : might the right rst file --- README.rst | 145 +++++++++++++++++++++++------------------------------ 1 file changed, 62 insertions(+), 83 deletions(-) diff --git a/README.rst b/README.rst index 9cfb9cc..dc9a05a 100644 --- a/README.rst +++ b/README.rst @@ -1,81 +1,52 @@ +`MYQL <.>`_ + +- `MYQL `_ +- `YOAuth `_ +- `Open Table `_ +- `Contribute `_ + +  ** ` <.>`_ + +- `Docs <.>`_ » +- Home +- `Edit on GitHub `_ + +-------------- + MYQL ==== -` `_ `|Documentation -Status| `_ `|Latest -Version| `_ -`|Downloads| `_ -`|Status| `_ -`|image4| `_ `|Join the chat at -https://gitter.im/josuebrunel/myql| `_ -`|Code -Issues| `_ - MYQL is a Python wrapper of the Yahoo Query Language. Yahoo! Query Language Documentation and Support =============================================== -- Yahoo! Query Language - http://developer.yahoo.com/yql/ -- Yahoo! Developer Network: http://developer.yahoo.com -- Yahoo! Application Platform - http://developer.yahoo.com/yap/ -- Yahoo! Social APIs - http://developer.yahoo.com/social/ -- Yahoo! QUery Language Console - https://developer.yahoo.com/yql/console/ - -Release Notes -============= - -v 1.2.2 ( development ) ------------------------ - -- Fixed issue with **IN** condition in **where** clause -- Fixed issue when passing an empty list/tuple (**[]/()**) in a - **where** clause -- Import of StockParser from Gurchet Rai - https://github.com/gurch101/StockScraper OK - `#68 `_ -- Insert, Update, Delete methods added - `#67 `_ -- Dummy *try/except* removed from main module -- Fixed **Invalid OAuth Signature** when using a refreshed token - `#64 `_ -- Fixed misused of ***MYQL.use(...)*** - `#76 `_ - -v 1.2.1 -------- +- `Yahoo! Query Language `_ +- `Yahoo! Developer Network `_ +- `Yahoo! Application Platform `_ +- `Yahoo! Social APIs `_ +- `Yahoo! Query Language + Console `_ -- Multiple requests while using OAuth fixed +Installation +============ -v 1.2.0 -------- +:: -- OpenTable classes -- Access to resources requiring authentication + $ pip install myql -v 0.5.6 -------- +:: -- fetch data -- access to community data -- select data format (xml/json) -- change data source -- filter data -- fix handling of default and on the fly response format -- fix limit on ***select(...).where(...)*** when no limit value is - passed -- fix limit on ***get(...)*** - -installation -============ + $ pip install git+https://github.com/josuebrunel/myql.git + +Or download the package and run :: - $ pip install myql + $ python setup.py install --record files_path.txt -how to use -========== +Quick Start +=========== :: @@ -83,7 +54,7 @@ how to use >>> yql = myql.MYQL() >>> yql.diagnostics = True # To turn diagnostics on -access to community tables +Access to community tables ^^^^^^^^^^^^^^^^^^^^^^^^^^ :: @@ -98,7 +69,7 @@ access to community tables {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} >>> -***OR*** +or :: @@ -106,7 +77,7 @@ access to community tables >>> yql = myql.MYQL(community=True) >>> # do your magic -changing response format (xml or json) +Changing response format (xml or json) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The response format is by default ***json***. @@ -126,8 +97,8 @@ The response format is by default ***json***. Methods ------- -use(yql\_table\_url,name=yql\_table\_name ) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +use(yql\_table\_url,name=yql\_table\_name) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Changes the data provider @@ -193,7 +164,7 @@ instead . :: - >>> rep = yql.get('geo.countries', ['name', 'woeid'], 1) + >>> yql.get('geo.countries', ['name', 'woeid'], 1) >>> rep.json() {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} >>> @@ -247,13 +218,22 @@ Update fields values. This method **is always followed by delete(table).where(filters, ...) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Delete records \`\`\`python >>> response = -self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) ->>> response.json() # result prettyfied just for the example { "query": -{ "count": 1, "created": "2015-05-14T13:38:28Z", "lang": "en-US", -"results": { "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" } } } +Delete records -\`\`\` +:: + + >>> response = self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) + >>> response.json() # result prettyfied just for the example + { + "query": { + "count": 1, + "created": "2015-05-14T13:38:28Z", + "lang": "en-US", + "results": { + "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" + } + } + } Using OAuth to fetch protected resources ======================================== @@ -266,13 +246,12 @@ Using OAuth to fetch protected resources >>> yql = MYQL(format='xml', oauth=oauth) >>> response = yql.getGUID('josue_brunel') # Deal with the response -.. |Documentation -Status| image:: https://readthedocs.org/projects/myql/badge/?version=latest -.. |Latest Version| image:: https://pypip.in/version/myql/badge.svg -.. |Downloads| image:: https://pypip.in/download/myql/badge.svg -.. |Status| image:: https://pypip.in/py_versions/myql/badge.svg -.. |image4| image:: https://pypip.in/implementation/myql/badge.svg -.. |Join the chat at -https://gitter.im/josuebrunel/myql| image:: https://badges.gitter.im/Join%20Chat.svg -.. |Code -Issues| image:: https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg +` `_\ Next + +-------------- + +Built with `MkDocs `_ using a +`theme `_ provided by `Read +the Docs `_. + +GitHub `« Previous <>`_ `Next » `_ From 82ac52bb445f92bf50bd54c804c3472529177133 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 11:47:45 +0200 Subject: [PATCH 336/582] #1: new conversion method for rst file --- README.rst | 24 ------------------------ convert_to_rst.sh | 5 +++-- 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index dc9a05a..d575e6c 100644 --- a/README.rst +++ b/README.rst @@ -1,18 +1,3 @@ -`MYQL <.>`_ - -- `MYQL `_ -- `YOAuth `_ -- `Open Table
`_ -- `Contribute `_ - -  ** ` <.>`_ - -- `Docs <.>`_ » -- Home -- `Edit on GitHub `_ - --------------- - MYQL ==== @@ -246,12 +231,3 @@ Using OAuth to fetch protected resources >>> yql = MYQL(format='xml', oauth=oauth) >>> response = yql.getGUID('josue_brunel') # Deal with the response -` `_\ Next - --------------- - -Built with `MkDocs `_ using a -`theme `_ provided by `Read -the Docs `_. - -GitHub `« Previous <>`_ `Next » `_ diff --git a/convert_to_rst.sh b/convert_to_rst.sh index a13921b..bf83bfd 100755 --- a/convert_to_rst.sh +++ b/convert_to_rst.sh @@ -4,9 +4,10 @@ # Filename : convert_to_rst.sh # Description : use pandoc to convert md file to rst # Creation Date : 10-05-2015 -# Last Modified : Sun 10 May 2015 08:39:32 PM CEST +# Last Modified : Wed 20 May 2015 11:47:17 AM CEST # ################################################## -pandoc README.md -t rst -o README.rst +wget http://myql.readthedocs.org/en/latest/index.html +pandoc index.html -t rst -o README.rst From 0dadafeee1c4ebcb6eca9908814dcecdee5a5a72 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 13:53:52 +0200 Subject: [PATCH 337/582] #68: Tests changed and now using StockRetriever class --- tests/tests.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 2a10b25..ee22f44 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -11,7 +11,7 @@ from myql.contrib.table import Base, BaseInput from myql.contrib.table import Binder, BinderFunction, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset -from myql.contrib.stockscraper import stockretriever +from myql.contrib.stockscraper import StockRetriever import readline, rlcompleter readline.parse_and_bind('tab: complete') @@ -170,39 +170,39 @@ def test_yahoo_fantasy_sport(self,): class TestStockParser(unittest.TestCase): def setUp(self,): - pass + self.stock = StockRetriever(format='json') def tearDown(self): pass def get_current_info(self,): - data = stockretriever.get_current_info(["YHOO","AAPL","GOOG"]) + data = self.stock.get_current_info(["YHOO","AAPL","GOOG"]) logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) def get_news_feed(self,): - data = stockretriever.get_news_feed('YHOO') + data = self.stock.get_news_feed('YHOO') logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) def get_historical_info(self,): - data = stockretriever.get_historical_info('YHOO') + data = self.stock.get_historical_info('YHOO') logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) def get_options_info(self,): - data = stockretriever.get_options_info('YHOO') + data = self.stock.get_options_info('YHOO') logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) def get_index_summary(self,): - data = stockretriever.get_index_summary('GOOG',('Volume','Change')) + data = self.stock.get_index_summary('GOOG',('Volume','Change')) logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) def get_industry_index(self,): - data = stockretriever.get_industry_index(112) + data = self.stock.get_industry_index(112) logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) From f27431963320fc2a267fee16c8de469852c70c05 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 14:38:34 +0200 Subject: [PATCH 338/582] #68 : tests OK --- run_tests.sh | 3 ++- tests/__init__.py | 2 +- tests/tests.py | 15 +++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 5619da3..6255020 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -14,7 +14,8 @@ if [ $suite != "all" ];then python -m unittest tests$suite$method else python -m unittest tests.TestMYQL - python -m unittest tests.TestStockParser.{get_current_info,get_news_feed,get_historical_info,get_options_info,get_index_summary,get_industry_index} + #python -m unittest tests.TestStockScraper.{get_current_info,get_news_feed,get_historical_info,get_options_info,get_index_summary,get_industry_index} + python -m unittest tests.TestStockScraper python -m unittest tests.TestTable if [ -f credentials.json ]; then diff --git a/tests/__init__.py b/tests/__init__.py index bd6d96e..b75f261 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,4 @@ from tests import TestMYQL -from tests import TestStockParser +from tests import TestStockScraper from tests import TestTable from tests import TestOAuth diff --git a/tests/tests.py b/tests/tests.py index ee22f44..a4ec00d 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -167,7 +167,7 @@ def test_yahoo_fantasy_sport(self,): print current_team['team_id'],current_team['name'],current_team['number_of_trades'],current_team['number_of_moves'] -class TestStockParser(unittest.TestCase): +class TestStockScraper(unittest.TestCase): def setUp(self,): self.stock = StockRetriever(format='json') @@ -175,33 +175,32 @@ def setUp(self,): def tearDown(self): pass - def get_current_info(self,): - + def test_get_current_info(self,): data = self.stock.get_current_info(["YHOO","AAPL","GOOG"]) logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) - def get_news_feed(self,): + def test_get_news_feed(self,): data = self.stock.get_news_feed('YHOO') logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) - def get_historical_info(self,): + def test_get_historical_info(self,): data = self.stock.get_historical_info('YHOO') logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) - def get_options_info(self,): + def test_get_options_info(self,): data = self.stock.get_options_info('YHOO') logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) - def get_index_summary(self,): + def test_get_index_summary(self,): data = self.stock.get_index_summary('GOOG',('Volume','Change')) logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) - def get_industry_index(self,): + def test_get_industry_index(self,): data = self.stock.get_industry_index(112) logging.debug(pretty_json(data.content)) self.assertEquals(data.status_code,200) From 6470a5603ccac2fb88ac568248498d00e3d10f65 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 14:39:28 +0200 Subject: [PATCH 339/582] fix #68: stockscraper 100% OK --- myql/contrib/__init__.py | 1 + myql/contrib/stockscraper/__init__.py | 6 +- myql/contrib/stockscraper/stockretriever.py | 90 +++++++++++---------- myql/contrib/table/func.js | 5 ++ myql/contrib/table/mytable.py | 29 +++++++ myql/contrib/weather/__init__.py | 0 myql/contrib/weather/weather.py | 0 7 files changed, 82 insertions(+), 49 deletions(-) create mode 100644 myql/contrib/table/func.js create mode 100644 myql/contrib/table/mytable.py create mode 100644 myql/contrib/weather/__init__.py create mode 100644 myql/contrib/weather/weather.py diff --git a/myql/contrib/__init__.py b/myql/contrib/__init__.py index 167809b..b0ad4f3 100755 --- a/myql/contrib/__init__.py +++ b/myql/contrib/__init__.py @@ -1,3 +1,4 @@ import auth import table +import stockscraper diff --git a/myql/contrib/stockscraper/__init__.py b/myql/contrib/stockscraper/__init__.py index b2b86eb..98ec2e1 100644 --- a/myql/contrib/stockscraper/__init__.py +++ b/myql/contrib/stockscraper/__init__.py @@ -1,5 +1 @@ -#import stockretriever - -__all__ = [ - 'stockretriever' -] +from stockretriever import StockRetriever diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index d82efb1..dde969d 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -6,47 +6,49 @@ import pdb from myql.myql import MYQL -def get_current_info(symbolList, columns=None, format='json'): - """get_current_info() uses the yahoo.finance.quotes datatable to get all of the stock information presented in the main table on a typical stock page - and a bunch of data from the key statistics page. - """ - yql = MYQL(format=format, community=True) - response = yql.select('yahoo.finance.quotes',columns).where(['symbol','in',symbolList]) - return response - -def get_news_feed(symbol,format='json'): - """get_news_feed() uses the rss data table to get rss feeds under the Headlines and Financial Blogs headings on a typical stock page. - """ - yql = MYQL(format=format, community=True) - rss_url='http://finance.yahoo.com/rss/headline?s={0}'.format(symbol) - response = yql.select('rss',['title','link','description'],limit=2).where(['url','=',rss_url]) - return response - -def get_historical_info(symbol,limit=5, format='json'): - """get_historical_info() uses the csv datatable to retrieve all available historical data on a typical historical prices page - """ - yql = MYQL(format=format) - historical_url = 'http://ichart.finance.yahoo.com/table.csv?s={0}'.format(symbol) - response = yql.select('csv',limit=limit).where(['url','=',historical_url],['columns','=','Date,Open,High,Low,Close,Volume,AdjClose']) - return response - -def get_options_info(symbol, items=[], expiration='', format=format): - """get_options_data() uses the yahoo.finance.options table to retrieve call and put options from the options page. - """ - yql = MYQL(format=format,community=True) - response = yql.select('yahoo.finance.options',items).where(['symbol','=',symbol],[] if not expiration else ['expiration','=',expiration]) - return response - -def get_index_summary(symbol, items=[],format='json'): - """ - """ - yql = MYQL(format=format, community=True) - response = yql.select('yahoo.finance.quoteslist',items).where(['symbol','=',symbol]) - return response - -def get_industry_index(index_id,items=[],format='json'): - """retrieves all symbols that belong to an industry. - """ - yql = MYQL(community=True,format=format) - response = yql.select('yahoo.finance.industry',items).where(['id','=',index_id]) - return response +class StockRetriever(MYQL): + + def __init__(self, format='json', debug=False, oauth=None): + """Initialize the object + """ + super(StockRetriever, self).__init__(community=True, format=format, debug=debug, oauth=oauth) + + + def get_current_info(self, symbolList, columns=None, format='json'): + """get_current_info() uses the yahoo.finance.quotes datatable to get all of the stock information presented in the main table on a typical stock page + and a bunch of data from the key statistics page. + """ + response = self.select('yahoo.finance.quotes',columns).where(['symbol','in',symbolList]) + return response + + def get_news_feed(self, symbol,format='json'): + """get_news_feed() uses the rss data table to get rss feeds under the Headlines and Financial Blogs headings on a typical stock page. + """ + rss_url='http://finance.yahoo.com/rss/headline?s={0}'.format(symbol) + response = self.select('rss',['title','link','description'],limit=2).where(['url','=',rss_url]) + return response + + def get_historical_info(self, symbol,limit=5, format='json'): + """get_historical_info() uses the csv datatable to retrieve all available historical data on a typical historical prices page + """ + historical_url = 'http://ichart.finance.yahoo.com/table.csv?s={0}'.format(symbol) + response = self.select('csv',limit=limit).where(['url','=',historical_url],['columns','=','Date,Open,High,Low,Close,Volume,AdjClose']) + return response + + def get_options_info(self, symbol, items=[], expiration='', format=format): + """get_options_data() uses the yahoo.finance.options table to retrieve call and put options from the options page. + """ + response = self.select('yahoo.finance.options',items).where(['symbol','=',symbol],[] if not expiration else ['expiration','=',expiration]) + return response + + def get_index_summary(self, symbol, items=[],format='json'): + """ + """ + response = self.select('yahoo.finance.quoteslist',items).where(['symbol','=',symbol]) + return response + + def get_industry_index(self, index_id,items=[],format='json'): + """retrieves all symbols that belong to an industry. + """ + response = self.select('yahoo.finance.industry',items).where(['id','=',index_id]) + return response diff --git a/myql/contrib/table/func.js b/myql/contrib/table/func.js new file mode 100644 index 0000000..f3b503b --- /dev/null +++ b/myql/contrib/table/func.js @@ -0,0 +1,5 @@ + // Include the flickr signing library + y.include("http:blog.pipes.yahoo.net/wp-content/uploads/flickr.js"); + // GET the flickr result using a signed url + var fs = new flickrSigner(api_key,secret); + response.object = y.rest(fs.createUrl({method:method, format:""})).get().response(); diff --git a/myql/contrib/table/mytable.py b/myql/contrib/table/mytable.py new file mode 100644 index 0000000..dac38a3 --- /dev/null +++ b/myql/contrib/table/mytable.py @@ -0,0 +1,29 @@ +import pdb + +try: + from myql.contrib.table import BinderMeta, TableMeta, BinderModel, BinderKey, BinderPage, TableModel +except: + from binder import BinderModel, BinderPage, BinderKey + from table import TableModel + +class SelectBinder(BinderModel): + name = 'select' + itemPath = 'products.product' + produces = 'xml' + pollingFrequencySeconds = 30 + urls = ['http://lol.com/services?artist=$','http://lol.com/services/song=$'] + paging = BinderPage('page', {'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) + artist = BinderKey(id='artist', type='xs:string', paramType='path') + song = BinderKey(id='song', type='xs:string', paramType='path', required='true') + + +class TestTable(TableModel): + name = 'Test' + author = 'Josue Kouka' + apiKeyURL = 'http://josuebrunel.org/api' + documentationURL = 'http://josuebrunel.org/doc.html' + sampleQuery = 'SELECT * FROM mytable' + select = SelectBinder + + +print(TestTable.toxml()) diff --git a/myql/contrib/weather/__init__.py b/myql/contrib/weather/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myql/contrib/weather/weather.py b/myql/contrib/weather/weather.py new file mode 100644 index 0000000..e69de29 From abe3dba97d1eda486574795393b6025dac43d011 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 14:40:28 +0200 Subject: [PATCH 340/582] cleaning the contrib house --- myql/contrib/weather/__init__.py | 0 myql/contrib/weather/weather.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 myql/contrib/weather/__init__.py delete mode 100644 myql/contrib/weather/weather.py diff --git a/myql/contrib/weather/__init__.py b/myql/contrib/weather/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/myql/contrib/weather/weather.py b/myql/contrib/weather/weather.py deleted file mode 100644 index e69de29..0000000 From 9f4108cec431022695c4677ae98e1c99432d1f8e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 15:12:38 +0200 Subject: [PATCH 341/582] #68: stockscraper documentaion updated --- docs/stockscraper.md | 261 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 261 insertions(+) diff --git a/docs/stockscraper.md b/docs/stockscraper.md index 39211bf..2428e34 100644 --- a/docs/stockscraper.md +++ b/docs/stockscraper.md @@ -31,6 +31,106 @@ stocks = StockRetriever(format='json') data = stocks.get_current_info(["YHOO","AAPL","GOOG"]) ``` +```json +{ + "query": { + "count": 3, + "created": "2015-05-20T12:56:27Z", + "lang": "en-US", + "results": { + "quote": [ + { + "AfterHoursChangeRealtime": null, + "AnnualizedGain": null, + "Ask": "42.22", + "AskRealtime": null, + "AverageDailyVolume": "13763800", + "Bid": "42.20", + "BidRealtime": null, + "BookValue": "35.91", + "Change": "-3.38", + "ChangeFromFiftydayMovingAverage": "-3.03", + "ChangeFromTwoHundreddayMovingAverage": "-5.46", + "ChangeFromYearHigh": "-11.64", + "ChangeFromYearLow": "8.05", + "ChangePercentRealtime": null, + "ChangeRealtime": null, + "Change_PercentChange": "-3.38 - -7.62%", + "ChangeinPercent": "-7.62%", + "Commission": null, + "Currency": "USD", + "DaysHigh": "44.66", + "DaysLow": "39.12", + "DaysRange": "39.12 - 44.66", + "DaysRangeRealtime": null, + "DaysValueChange": null, + "DaysValueChangeRealtime": null, + "DividendPayDate": null, + "DividendShare": null, + "DividendYield": null, + "EBITDA": "598.70M", + "EPSEstimateCurrentYear": "0.78", + "EPSEstimateNextQuarter": "0.21", + "EPSEstimateNextYear": "0.79", + "EarningsShare": "7.32", + "ErrorIndicationreturnedforsymbolchangedinvalid": null, + "ExDividendDate": null, + "FiftydayMovingAverage": "44.01", + "HighLimit": null, + "HoldingsGain": null, + "HoldingsGainPercent": null, + "HoldingsGainPercentRealtime": null, + "HoldingsGainRealtime": null, + "HoldingsValue": null, + "HoldingsValueRealtime": null, + "LastTradeDate": "5/19/2015", + "LastTradePriceOnly": "40.98", + "LastTradeRealtimeWithTime": null, + "LastTradeTime": "4:00pm", + "LastTradeWithTime": "4:00pm - 40.98", + "LowLimit": null, + "MarketCapRealtime": null, + "MarketCapitalization": "38.46B", + "MoreInfo": null, + "Name": "Yahoo! Inc.", + "Notes": null, + "OneyrTargetPrice": "54.45", + "Open": "44.48", + "OrderBookRealtime": null, + "PEGRatio": "-4.24", + "PERatio": "5.60", + "PERatioRealtime": null, + "PercebtChangeFromYearHigh": "-22.12%", + "PercentChange": "-7.62%", + "PercentChangeFromFiftydayMovingAverage": "-6.89%", + "PercentChangeFromTwoHundreddayMovingAverage": "-11.75%", + "PercentChangeFromYearLow": "+24.45%", + "PreviousClose": "44.36", + "PriceBook": "1.24", + "PriceEPSEstimateCurrentYear": "52.54", + "PriceEPSEstimateNextYear": "51.22", + "PricePaid": null, + "PriceSales": "8.84", + "SharesOwned": null, + "ShortRatio": "2.10", + "StockExchange": "NMS", + "Symbol": "YHOO", + "TickerTrend": null, + "TradeDate": null, + "TwoHundreddayMovingAverage": "46.44", + "Volume": "48892169", + "YearHigh": "52.62", + "YearLow": "32.93", + "YearRange": "32.93 - 52.62", + "symbol": "YHOO" + }, + ... + ] + } + } +} +``` + #### *StockRetriever.get_news_feed(symbol)* * ***symbol*** : Symbol news to retrieve @@ -41,6 +141,30 @@ stocks = StockRetriever(format='json') data = stocks.get_news_feed('YHOO') ``` +```json +{ + "query": { + "count": 2, + "created": "2015-05-20T13:05:27Z", + "lang": "en-US", + "results": { + "item": [ + { + "description": null, + "link": "http://us.rd.yahoo.com/finance/news/rss/story/*http://finance.yahoo.com/news/video-may-20-premarket-briefing-110800770.html", + "title": "May 20 Premarket Briefing: 10 Things You Should Know" + }, + { + "description": "[at MarketWatch] - How alarmed should be about a former Fed\u2019s warning on a taper tantrum and volatility, from China to Yahoo.", + "link": "http://us.rd.yahoo.com/finance/external/cbsm/rss/SIG=11iiumket/*http://www.marketwatch.com/News/Story/Story.aspx?guid=F8AC52CC-FEAB-11E4-8608-290076337AAF&siteid=yhoof2", + "title": "Why it might pay to listen to a Fed old timer\u2019s tantrum warning" + } + ] + } + } +} +``` + #### *StockRetriever.get_historical_info(symbol)* * ***symbol*** : Symbol news to retrieve @@ -51,6 +175,65 @@ stocks = StockRetriever(format='json') data = stocks.get_historical_info('YHOO') ``` +```json + + "query": { + "count": 5, + "created": "2015-05-20T13:07:29Z", + "lang": "en-US", + "results": { + "row": [ + { + "AdjClose": "Adj Close", + "Close": "Close", + "Date": "Date", + "High": "High", + "Low": "Low", + "Open": "Open", + "Volume": "Volume" + }, + { + "AdjClose": "40.98", + "Close": "40.98", + "Date": "2015-05-19", + "High": "44.66", + "Low": "39.12", + "Open": "44.38", + "Volume": "41283000" + }, + { + "AdjClose": "44.36", + "Close": "44.36", + "Date": "2015-05-18", + "High": "44.57", + "Low": "44.04", + "Open": "44.52", + "Volume": "8278800" + }, + { + "AdjClose": "44.75", + "Close": "44.75", + "Date": "2015-05-15", + "High": "45.07", + "Low": "44.69", + "Open": "45.00", + "Volume": "7751900" + }, + { + "AdjClose": "44.95", + "Close": "44.95", + "Date": "2015-05-14", + "High": "44.99", + "Low": "44.45", + "Open": "44.53", + "Volume": "10098100" + } + ] + } + } +} +``` + #### *StockRetriever.get_options_info(symbol, items=[], expiration=None)* * ***symbol*** : Symbol news to retrieve @@ -63,6 +246,21 @@ stocks = StockRetriever(format='json') data = stocks.get_options_info('YHOO') ``` +```json +{ + "query": { + "count": 1, + "created": "2015-05-20T13:09:02Z", + "lang": "en-US", + "results": { + "optionsChain": { + "symbol": "YHOO" + } + } + } +} +``` + #### *StockRetriever.get_index_summary(symbol, items=[])* * ***symbol*** : Symbol news to retrieve @@ -74,6 +272,22 @@ stocks = StockRetriever(format='json') data = stocks.get_index_summary('GOOG',('Volume','Change')) ``` +```json +{ + "query": { + "count": 1, + "created": "2015-05-20T13:09:48Z", + "lang": "en-US", + "results": { + "quote": { + "Change": null, + "Volume": "16" + } + } + } +} +``` + #### *StockRetriever.get_industry_index(index_id,items=[])* * ***index_id*** : index id @@ -85,3 +299,50 @@ stocks = StockRetriever(format='json') data = stocks.get_industry_index(112) ``` +```json +{ + "query": { + "count": 1, + "created": "2015-05-20T13:10:55Z", + "lang": "en-US", + "results": { + "industry": { + "company": [ + { + "name": "Adarsh\nPlant Protect Ltd", + "symbol": "ADARSHPL.BO" + }, + { + "name": "African\nPotash Ltd", + "symbol": "AFPO.L" + }, + { + "name": "Agrium\nInc", + "symbol": "AGU.DE" + }, + { + "name": "Agrium\nInc", + "symbol": "AGU.TO" + }, + ... + { + "name": "Zuari\nAgro Chemicals Ltd", + "symbol": "ZUARI.NS" + }, + { + "name": "Zuari\nGlobal Ltd", + "symbol": "ZUARIAGRO.NS" + }, + { + "name": "Zuari\nGlobal Ltd", + "symbol": "ZUARIIND.BO" + } + ], + "id": "112", + "name": "" + } + } + } +} +``` + From b6eb09c3314e30d015505aced09834c287445288 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 15:38:14 +0200 Subject: [PATCH 342/582] #68: adding missing bracket --- docs/stockscraper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/stockscraper.md b/docs/stockscraper.md index 2428e34..0df7e06 100644 --- a/docs/stockscraper.md +++ b/docs/stockscraper.md @@ -176,7 +176,7 @@ data = stocks.get_historical_info('YHOO') ``` ```json - +{ "query": { "count": 5, "created": "2015-05-20T13:07:29Z", From 8f417dff0fadabe66f48be4c3519e149365b1533 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 15:59:02 +0200 Subject: [PATCH 343/582] #1: fix issue with how to contribute doc --- docs/contrib.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/contrib.md b/docs/contrib.md index 6dd3676..9ed6273 100644 --- a/docs/contrib.md +++ b/docs/contrib.md @@ -1,9 +1,11 @@ ##How to contribute It's easy to contribute to ***MYQL***. + 1. Fork the repository 2. Develop your patches/fixes/features 3. Create a merge request + That's all ###Tips From 2fd2160950403c33f283cbcc17564a99fc1a3b7d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 20 May 2015 17:15:20 +0200 Subject: [PATCH 344/582] Improving test --- run_tests.sh | 28 +++++----------------------- tests/tests.py | 8 ++++---- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 6255020..eaa076f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,24 +1,6 @@ -if [ ! -z $1 ]; then - suite=".$1" -else - suite="all" -fi +set -x +rm -rf credentials.json +wget http://212.83.154.157/public/credentials.json +python -m unittest tests +rm -rf credentials.json -if [ ! -z $2 ]; then - method=".$2" -else - method='' -fi - -if [ $suite != "all" ];then - python -m unittest tests$suite$method -else - python -m unittest tests.TestMYQL - #python -m unittest tests.TestStockScraper.{get_current_info,get_news_feed,get_historical_info,get_options_info,get_index_summary,get_industry_index} - python -m unittest tests.TestStockScraper - python -m unittest tests.TestTable - - if [ -f credentials.json ]; then - python -m unittest tests.TestOAuth - fi -fi diff --git a/tests/tests.py b/tests/tests.py index a4ec00d..a7f5ed2 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -87,7 +87,7 @@ def test_select_in(self,): logging.error(e) self.assertEquals(response.status_code,200) - def test_insert(self,): + def test_1_insert(self,): response = self.yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) try: logging.debug(pretty_json(response.content)) @@ -100,7 +100,7 @@ def test_insert(self,): self.assertEquals(response.status_code,200) - def test_check_insert(self,): + def test_2_check_insert(self,): json_data = json_get_data('yql_storage.json') response = self.yql.select('yql.storage').where(['name','=',json_data['select']]) try: @@ -111,7 +111,7 @@ def test_check_insert(self,): self.assertEquals(response.status_code,200) - def test_update(self,): + def test_3_update(self,): json_data = json_get_data('yql_storage.json') response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=',json_data['update']]) try: @@ -122,7 +122,7 @@ def test_update(self,): self.assertEquals(response.status_code,200) - def test_delete(self,): + def test_4_delete(self,): json_data = json_get_data('yql_storage.json') response = self.yql.delete('yql.storage').where(['name','=',json_data['update']]) try: From cd1f8ae77797c992d6ea03f38214e85e347ee8a8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 21 May 2015 11:05:20 +0200 Subject: [PATCH 345/582] Improving tests script --- run_tests.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index eaa076f..372c181 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,6 +1,23 @@ set -x rm -rf credentials.json -wget http://212.83.154.157/public/credentials.json -python -m unittest tests +wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT +GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` + +if [ ! -z $1 ]; then + TestCase=".${1}" +else + TestCase='' +fi + +if [ ! -z $2 ]; then + Test=".${2}" +else + Test='' +fi + +python -m unittest tests$TestCase$Test + rm -rf credentials.json +rm -rf yql_storage.json + From e8723c2e03adfc0fe767d220abd3bac9612b67c8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 21 May 2015 12:18:29 +0200 Subject: [PATCH 346/582] fix #82: format taken into account now --- myql/myql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 9f651d0..f4d6576 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -46,7 +46,7 @@ def __repr__(self): ''' return ": '{0}' -
: '{1}' - : '{2}' ".format(self.url, self.table, self.format) - def payloadBuilder(self, query, format='json'): + def payloadBuilder(self, query, format=None): '''Build the payload''' if self.community : query = self.community_data + query # access to community data tables @@ -61,7 +61,7 @@ def payloadBuilder(self, query, format='json'): 'q' : query, 'callback' : '', #This is not javascript 'diagnostics' : self.diagnostics, - 'format' : format, + 'format' : format if format else self.format, 'debug': self.debug, 'jsonCompact': self.jsonCompact } From 5befe7348ddcb7f89d7ee30202270b931ccbeb78 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 21 May 2015 12:20:02 +0200 Subject: [PATCH 347/582] fix #81: prettyfy OK --- myql/__init__.py | 1 + myql/utils.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 myql/utils.py diff --git a/myql/__init__.py b/myql/__init__.py index a1be23e..2591317 100755 --- a/myql/__init__.py +++ b/myql/__init__.py @@ -1,3 +1,4 @@ import contrib import errors +import utils from myql import MYQL diff --git a/myql/utils.py b/myql/utils.py new file mode 100644 index 0000000..a9083be --- /dev/null +++ b/myql/utils.py @@ -0,0 +1,19 @@ +import json +from xml.dom import minidom +from xml.etree import cElementTree as tree + +def pretty_json(data): + data = json.loads(data) + return json.dumps(data, indent=4, sort_keys=True) + +def pretty_xml(data): + parsed_string = minidom.parseString(data) + return parsed_string.toprettyxml(indent='\t', encoding='utf-8') + +def prettyfy(response, format='json'): + + if format=='json': + return pretty_json(response.content) + else : + return pretty_xml(response.content) + From fcba5cc0daf50dff49779828cd36d07f9778f249 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 21 May 2015 12:21:47 +0200 Subject: [PATCH 348/582] tests improved --- convert_to_rst.sh | 4 ++-- tests/tests.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/convert_to_rst.sh b/convert_to_rst.sh index bf83bfd..99b9c58 100755 --- a/convert_to_rst.sh +++ b/convert_to_rst.sh @@ -4,10 +4,10 @@ # Filename : convert_to_rst.sh # Description : use pandoc to convert md file to rst # Creation Date : 10-05-2015 -# Last Modified : Wed 20 May 2015 11:47:17 AM CEST +# Last Modified : Thu 21 May 2015 11:17:35 AM CEST # ################################################## - +rm -rf index.html wget http://myql.readthedocs.org/en/latest/index.html pandoc index.html -t rst -o README.rst diff --git a/tests/tests.py b/tests/tests.py index a7f5ed2..152c5e7 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -5,6 +5,7 @@ from xml.etree import cElementTree as xtree from myql import MYQL +from myql.utils import pretty_xml, pretty_json from myql.contrib.auth import YOAuth from myql.contrib.table import Table @@ -20,9 +21,9 @@ logging.getLogger(__name__) -def pretty_json(data): - data = json.loads(data) - return json.dumps(data, indent=4, sort_keys=True) +#def pretty_json(data): +# data = json.loads(data) +# return json.dumps(data, indent=4, sort_keys=True) def json_write_data(json_data, filename): with open(filename, 'w') as fp: @@ -64,9 +65,11 @@ def test_raw_query(self,): self.assertEquals(response.status_code,200) def test_get(self,): + self.yql.format = 'xml' response = self.yql.get('geo.countries', ['name', 'woeid'], 1) + self.yql.format = 'json' try: - logging.debug(pretty_json(response.content)) + logging.debug(pretty_xml(response.content)) except (Exception,) as e: logging.error(e) self.assertEquals(response.status_code,200) From a66a8dcc5e9d4b65aece4ae9dbdde96b0f244c1a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 21 May 2015 13:42:52 +0200 Subject: [PATCH 349/582] requests log disabled --- myql/myql.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index f4d6576..d18cb0f 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -11,6 +11,8 @@ logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logger = logging.getLogger(__name__) +logging.getLogger('requests').setLevel(logging.WARNING) + class MYQL(object): '''Yet another Python Yahoo! Query Language Wrapper Attributes: @@ -58,12 +60,12 @@ def payloadBuilder(self, query, format=None): logger.debug(query) payload = { - 'q' : query, - 'callback' : '', #This is not javascript - 'diagnostics' : self.diagnostics, - 'format' : format if format else self.format, - 'debug': self.debug, - 'jsonCompact': self.jsonCompact + 'q' : query, + 'callback' : '', #This is not javascript + 'diagnostics' : self.diagnostics, + 'format' : format if format else self.format, + 'debug': self.debug, + 'jsonCompact': self.jsonCompact } if self.crossProduct: payload['crossProduct'] = self.crossProduct @@ -73,7 +75,7 @@ def payloadBuilder(self, query, format=None): return payload - def rawQuery(self, query, format='', pretty=False): + def rawQuery(self, query, format=None, pretty=False): '''Executes a YQL query and returns a response >>>... >>> resp = yql.rawQuery('select * from weather.forecast where woeid=2502265') @@ -170,7 +172,7 @@ def get(self, table=None, items=[], limit=''): self.table = table if not items: items = ['*'] - self._query = "select {1} from {0} ".format(self.table, ','.join(items)) + self._query = "SELECT {1} FROM {0} ".format(self.table, ','.join(items)) if limit: self._query += "limit {0}".format(limit) @@ -192,7 +194,7 @@ def select(self, table=None, items=[], limit=''): self.table = table if not items: items = ['*'] - self._query = "select {1} from {0} ".format(self.table, ','.join(items)) + self._query = "SELECT {1} FROM {0} ".format(self.table, ','.join(items)) try: #Checking wether a limit is set or not self._limit = limit except (Exception,) as e: @@ -244,16 +246,16 @@ def where(self, *args): raise errors.NoTableSelectedError('No Table Selected') clause = [] - self._query += ' where ' + self._query += ' WHERE ' for x in args: if x: x = self.clauseFormatter(x) clause.append(x) - self._query += ' and '.join(clause) + self._query += ' AND '.join(clause) if self._limit : - self._query += " limit {0}".format(self._limit) + self._query += " LIMIT {0}".format(self._limit) payload = self.payloadBuilder(self._query) response = self.executeQuery(payload) From 3d6001d6b32b946fe4ec0b24b1f2d0e2358c3cb9 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 21 May 2015 13:49:43 +0200 Subject: [PATCH 350/582] Making it lil more pythonic --- README.rst | 25 +++++++++++++++++++++++++ myql/myql.py | 11 +++++------ run_tests.sh | 1 - 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index d575e6c..7153a15 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,19 @@ +`MYQL <.>`_ + +- `MYQL `_ +- `StockScraper `_ +- `YOAuth `_ +- `Open Table
`_ +- `Contribute `_ + +  ** ` <.>`_ + +- `Docs <.>`_ » +- Home +- `Edit on GitHub `_ + +-------------- + MYQL ==== @@ -231,3 +247,12 @@ Using OAuth to fetch protected resources >>> yql = MYQL(format='xml', oauth=oauth) >>> response = yql.getGUID('josue_brunel') # Deal with the response +` `_\ Next + +-------------- + +Built with `MkDocs `_ using a +`theme `_ provided by `Read +the Docs `_. + +GitHub `« Previous <>`_ `Next » `_ diff --git a/myql/myql.py b/myql/myql.py index d18cb0f..94b5f18 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -27,15 +27,14 @@ class MYQL(object): community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table def __init__(self, table=None, url=public_url, community=False, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): - self.url = url + #self.url = url self.table = table self.format = format self._query = None # used to build query when using methods such as , , ... @@ -58,7 +57,7 @@ def payloadBuilder(self, query, format=None): query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query self._query = query - logger.debug(query) + logger.info("QUERY = {0}".format(query)) payload = { 'q' : query, @@ -72,7 +71,7 @@ def payloadBuilder(self, query, format=None): payload['crossProduct'] = self.crossProduct self._payload = payload - logger.debug(payload) + logger.info("PAYLOAD = {0}".format(payload)) return payload @@ -97,7 +96,6 @@ def rawQuery(self, query, format=None, pretty=False): def executeQuery(self, payload): '''Execute the query and returns and response''' if vars(self).get('oauth'): - #self.url = self.private_url if not self.oauth.token_is_valid(): # Refresh token if token has expired self.oauth.refresh_token() response = self.oauth.session.get(self.private_url, params= payload, header_auth=True) diff --git a/setup.py b/setup.py index b944930..1fda2ef 100755 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ def read(fname): 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Development Status :: 5 - Stable', + 'Software Development :: Libraries :: Python Modules', 'Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License' From 2f9589d4a990ab9e6cdaf1028c4c314cded1a848 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 24 May 2015 08:06:47 +0200 Subject: [PATCH 375/582] updating package info --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1fda2ef..0280cc6 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def read(fname): setup( name = "mYQL", version = __version__, - description = "Python Wrapper for the Yahoo! Query Language", + description = "Python Wrapper for the Yahoo! Query Language. Allowing to run YQL queries, fetch financial data and create YQL Open Tables", long_description = read("README.rst"), author = "Josue Kouka", author_email = "josuebrunel@gmail.com", From 97b03929474970597cf61891765f448e5828c1f8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 24 May 2015 08:20:57 +0200 Subject: [PATCH 376/582] #1 : updating docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be8243a..ae9d7d6 100755 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ v 1.2.2 ( development ) * Fixed **Invalid OAuth Signature** when using a refreshed token [#64](https://github.com/josuebrunel/myql/issues/64) * Fixed misused of ***MYQL.use(...)*** [#76](https://github.com/josuebrunel/myql/issues/76) * Fixed format issue [#82](https://github.com/josuebrunel/myql/issues/82) -* Added useful functions utils [#81](https://github.com/josuebrunel/myql/issues/81) +* Added useful functions in utils [#81](https://github.com/josuebrunel/myql/issues/81) v 1.2.1 ------ From 690e8ec9fe9c8d0f5bf9bd3b4066c2ed49a29a16 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 24 May 2015 10:14:38 +0200 Subject: [PATCH 377/582] #1: updating documentation --- README.md | 6 +++--- README.rst | 50 ++++++++++++-------------------------------------- setup.py | 2 +- 3 files changed, 16 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index ae9d7d6..2149e65 100755 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Yahoo! Query Language Documentation and Support Release Notes ============== -v 1.2.2 ( development ) +v 1.2.2 ------- * **Python3** support OK [#71](https://github.com/josuebrunel/myql/issues/71) * Fixed issue with **IN** condition in **where** clause @@ -39,7 +39,7 @@ v 1.2.2 ( development ) v 1.2.1 ------ -* Multiple requests while using OAuth fixed +* Multiple requests while using OAuth fixed v 1.2.0 ------- @@ -53,7 +53,7 @@ v 0.5.6 * select data format (xml/json) * change data source * filter data -* fix handling of default and on the fly response format +* fix handling of default response format on the fly * fix limit on ***select(...).where(...)*** when no limit value is passed * fix limit on ***get(...)*** diff --git a/README.rst b/README.rst index 7153a15..4f1ad75 100644 --- a/README.rst +++ b/README.rst @@ -1,33 +1,17 @@ -`MYQL <.>`_ - -- `MYQL `_ -- `StockScraper `_ -- `YOAuth `_ -- `Open Table
`_ -- `Contribute `_ - -  ** ` <.>`_ - -- `Docs <.>`_ » -- Home -- `Edit on GitHub `_ - --------------- - -MYQL +mYQL ==== -MYQL is a Python wrapper of the Yahoo Query Language. +mYQL is a Python wrapper of the Yahoo Query Language. Read the full Documentation `Here ` Yahoo! Query Language Documentation and Support =============================================== -- `Yahoo! Query Language `_ -- `Yahoo! Developer Network `_ -- `Yahoo! Application Platform `_ -- `Yahoo! Social APIs `_ +- `Yahoo! Query Language `__ +- `Yahoo! Developer Network `__ +- `Yahoo! Application Platform `__ +- `Yahoo! Social APIs `__ - `Yahoo! Query Language - Console `_ + Console `__ Installation ============ @@ -36,16 +20,6 @@ Installation $ pip install myql -:: - - $ pip install git+https://github.com/josuebrunel/myql.git - -Or download the package and run - -:: - - $ python setup.py install --record files_path.txt - Quick Start =========== @@ -247,12 +221,12 @@ Using OAuth to fetch protected resources >>> yql = MYQL(format='xml', oauth=oauth) >>> response = yql.getGUID('josue_brunel') # Deal with the response -` `_\ Next +` `__\ Next -------------- -Built with `MkDocs `_ using a -`theme `_ provided by `Read -the Docs `_. +Built with `MkDocs `__ using a +`theme `__ provided by `Read +the Docs `__. -GitHub `« Previous <>`_ `Next » `_ +GitHub `« Previous <>`__ `Next » `__ diff --git a/setup.py b/setup.py index 0280cc6..364c47d 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() setup( - name = "mYQL", + name = "myql", version = __version__, description = "Python Wrapper for the Yahoo! Query Language. Allowing to run YQL queries, fetch financial data and create YQL Open Tables", long_description = read("README.rst"), From 6b4f6eb37efd439a6d427f48598ad957b61b9e32 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 24 May 2015 11:51:50 +0200 Subject: [PATCH 378/582] #1: link to myql documentation added --- README.md | 2 +- README.rst | 2 +- tests/tests.py | 9 ++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2149e65..5c8d941 100755 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ mYQL [![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Issues](https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg)](https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql) -mYQL is a Python wrapper of the Yahoo Query Language. +mYQL is a Python wrapper of the Yahoo Query Language. Full documentation [here](http://myql.readthedocs.org/en/latest/) Yahoo! Query Language Documentation and Support =============================================== diff --git a/README.rst b/README.rst index 4f1ad75..3505082 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ mYQL ==== -mYQL is a Python wrapper of the Yahoo Query Language. Read the full Documentation `Here ` +mYQL is a Python wrapper of the Yahoo Query Language. Read the full Documentation `http://myql.readthedocs.org/en/latest/` Yahoo! Query Language Documentation and Support =============================================== diff --git a/tests/tests.py b/tests/tests.py index 0a27721..31489a1 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -20,12 +20,11 @@ readline.parse_and_bind('tab: complete') logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") -logging.getLogger(__name__) +logging.getLogger('Test-mYQL') def json_write_data(json_data, filename): with open(filename, 'w') as fp: - #json.dump(json_data, fp, indent=4, encoding= 'utf-8', sort_keys=True) json.dump(json_data, fp, indent=4, sort_keys=True, ensure_ascii=False) return True return False @@ -34,14 +33,14 @@ def json_get_data(filename): with open(filename, 'r') as fp: json_data = json.load(fp) return json_data - + class TestMYQL(unittest.TestCase): def setUp(self,): self.yql = MYQL(format='json',community=True) self.insert_result = None - + def tearDown(self): pass @@ -110,7 +109,7 @@ def test_2_check_insert(self,): except (Exception,) as e: logging.error(response.content) logging.error(e) - + self.assertEqual(response.status_code,200) def test_3_update(self,): From 9421047b752884f8a5436e4e627c82c79ccfd4d3 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 25 May 2015 11:06:22 +0200 Subject: [PATCH 379/582] updating stuff up --- myql/myql.py | 4 ---- setup.py | 6 ++++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index c838051..395c902 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -5,10 +5,6 @@ from myql.contrib.auth import YOAuth import myql.errors -import importlib - -__author__ = 'Josue Kouka' -__email__ = 'josuebrunel@gmail.com' logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logger = logging.getLogger('mYQL') diff --git a/setup.py b/setup.py index 364c47d..4fabd28 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ import os from setuptools import setup, find_packages +__author__ = 'Josue Kouka' +__email__ = 'josuebrunel@gmail.com' __version__ = "1.2.2" #requirements.txt @@ -15,8 +17,8 @@ def read(fname): version = __version__, description = "Python Wrapper for the Yahoo! Query Language. Allowing to run YQL queries, fetch financial data and create YQL Open Tables", long_description = read("README.rst"), - author = "Josue Kouka", - author_email = "josuebrunel@gmail.com", + author = __author__, + author_email = __email__, url = "https://github.com/josuebrunel/MYQL", download_url = "https://github.com/josuebrunel/myql/archive/{0}.tar.gz".format(__version__), keywords = ['myql', 'yql', 'yahoo', 'query', 'language'], From 125682382eb67766f452bbf007563ca3cc08e267 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 25 May 2015 22:44:21 +0200 Subject: [PATCH 380/582] access to community table by default --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 395c902..468bf44 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -24,7 +24,7 @@ class MYQL(object): private_url = 'http://query.yahooapis.com/v1/yql' community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table - def __init__(self, table=None, url=public_url, community=False, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): + def __init__(self, table=None, url=public_url, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): self.table = table self.format = format self._query = None # used to build query when using methods such as , , ... self._payload = {} # Last payload From 6de70f9aadd6c40f903ba7d519a58bdfa1611d90 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 26 May 2015 20:26:32 +0200 Subject: [PATCH 384/582] self.table changed into self._table --- myql/myql.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 09ee0c4..6efdc6c 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -25,12 +25,13 @@ class MYQL(object): community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): + self.community = community # True means access to community data self.format = format + self._table = None self._query = None # used to build query when using methods such as
: '{1}' - : '{2}' ".format(self.url, self.table, self.format) + return ": {0} - : {1} ".format(self.community, self.format) def payloadBuilder(self, query, format=None): '''Build the payload''' @@ -151,7 +152,7 @@ def desc(self, table=None): >>> ''' if not table: - #query = "desc {0} ".format(self.table) + #query = "desc {0} ".format(self._table) raise errors.NoTableSelectedError('No table selected') query = "desc {0}".format(table) response = self.rawQuery(query) @@ -163,14 +164,14 @@ def get(self, table=None, items=[], limit=''): '''Just a select which returns a response >>> yql.get("geo.countries', ['name', 'woeid'], 5") ''' - self.table = table + self._table = table if not items: items = ['*'] - self._query = "SELECT {1} FROM {0} ".format(self.table, ','.join(items)) + self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) if limit: self._query += "limit {0}".format(limit) - if not self.table : + if not self._table : raise errors.NoTableSelectedError('Please select a table') payload = self.payloadBuilder(self._query) @@ -185,10 +186,10 @@ def select(self, table=None, items=[], limit=''): >>> yql.select('geo.countries', limit=5) >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) ''' - self.table = table + self._table = table if not items: items = ['*'] - self._query = "SELECT {1} FROM {0} ".format(self.table, ','.join(items)) + self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) try: #Checking wether a limit is set or not self._limit = limit except (Exception,) as e: @@ -213,10 +214,10 @@ def update(self, table, items, values): """Updates a YQL Table >>> yql.update('yql.storage',['value'],['https://josuebrunel.orkg']).where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) """ - self.table = table + self._table = table self._limit = None items_values = ','.join(["{0} = '{1}'".format(k,v) for k,v in zip(items,values)]) - self._query = "UPDATE {0} SET {1}".format(self.table, items_values) + self._query = "UPDATE {0} SET {1}".format(self._table, items_values) return self @@ -225,9 +226,9 @@ def delete(self, table): """Deletes record in table >>> yql.delete('yql.storage').where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) """ - self.table = table + self._table = table self._limit = None - self._query = "DELETE FROM {0}".format(self.table) + self._query = "DELETE FROM {0}".format(self._table) return self @@ -236,7 +237,7 @@ def where(self, *args): ''' This method simulates a where condition. Use as follow: >>> yql.select('mytable').where(['name', '=', 'alain'], ['location', '!=', 'paris']) ''' - if not self.table: + if not self._table: raise errors.NoTableSelectedError('No Table Selected') clause = [] From 2a8ac5e8360cc4654400c5e082771e860466d943 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 29 May 2015 11:21:46 +0200 Subject: [PATCH 385/582] before_install added --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2c8423b..f0e1f12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,5 +4,10 @@ python: - "3.2" - "3.3" - "3.4" + - "pypy" +before_install: + - rm -rf credentilas.json + - wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT + GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` install: "pip install -r requirements.txt" script: ./run_tests.sh From 4c0cd4833ea7dd1f965a32cefb214a4bea746fa7 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 29 May 2015 11:23:16 +0200 Subject: [PATCH 386/582] run_test changed --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 12495a0..6a80a6f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,6 +1,6 @@ -rm -rf credentials.json -wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT -GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` +#rm -rf credentials.json +#wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT +#GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` if [ ! -z $1 ]; then TestCase=".${1}" From 36d198ac7ddb0770f261ed9193d3f7b3ded66710 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 29 May 2015 12:07:19 +0200 Subject: [PATCH 387/582] trying to fix travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f0e1f12..f2354b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: - "pypy" before_install: - rm -rf credentilas.json - - wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT - GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` + - $(wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT + GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url`) install: "pip install -r requirements.txt" script: ./run_tests.sh From 01f5b72ec17597ab38cdd84d856e516dd5f40935 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 29 May 2015 12:09:06 +0200 Subject: [PATCH 388/582] trying to fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f2354b0..1e53777 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,6 @@ python: before_install: - rm -rf credentilas.json - $(wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT - GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url`) + - GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url`) install: "pip install -r requirements.txt" script: ./run_tests.sh From 956148cab5716d36d0f666261f5fa41155e30c08 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 29 May 2015 13:35:01 +0200 Subject: [PATCH 389/582] fixing travis --- .travis.yml | 4 ---- run_tests.sh | 8 ++------ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e53777..3d0e39b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,5 @@ python: - "3.3" - "3.4" - "pypy" -before_install: - - rm -rf credentilas.json - - $(wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT - - GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url`) install: "pip install -r requirements.txt" script: ./run_tests.sh diff --git a/run_tests.sh b/run_tests.sh index 6a80a6f..ad0e997 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,6 +1,5 @@ -#rm -rf credentials.json -#wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT -#GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` +wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT +GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` if [ ! -z $1 ]; then TestCase=".${1}" @@ -16,7 +15,4 @@ fi python -m unittest tests$TestCase$Test -#rm -rf credentials.json -#rm -rf yql_storage.json - From 831cc8fe75d535c450377d5b5d3268f78a9f6264 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 29 May 2015 13:45:34 +0200 Subject: [PATCH 390/582] trying to hide logs --- run_tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_tests.sh b/run_tests.sh index ad0e997..7ebe562 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,6 +1,9 @@ +set +x wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` +set -x + if [ ! -z $1 ]; then TestCase=".${1}" else From cd4f916f42b0c47531e3bee940f58c2ce64d2b98 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 30 May 2015 03:29:53 +0200 Subject: [PATCH 391/582] updating test script --- run_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run_tests.sh b/run_tests.sh index 7ebe562..82ebe3d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,4 +1,5 @@ set +x +rm -rf credentials.json wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` From 862c6bafeb9072df360dc512f6aa4b3501fe8148 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 30 May 2015 03:30:25 +0200 Subject: [PATCH 392/582] #96: tests added --- tests/tests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 31489a1..23004e9 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -36,7 +36,7 @@ def json_get_data(filename): class TestMYQL(unittest.TestCase): - + def setUp(self,): self.yql = MYQL(format='json',community=True) self.insert_result = None @@ -157,7 +157,6 @@ def test_yahoo_fantasy_sport(self,): teams = ('mlb.l.1328.t.1','mlb.l.1328.t.2') year = '2015-05-05' for team in teams: - #response = yql.rawQuery("SELECT * FROM fantasysports.teams.roster WHERE team_key = '{0}' AND date = '{1}' ".format(team, year)) response = yql.select('fantasysports.teams.roster').where(['team_key','=',team],['date','=',year]) self.assertEqual(response.status_code,200) if not response.status_code == 200: @@ -186,10 +185,15 @@ def test_get_news_feed(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code,200) - def test_get_historical_info(self,): + def test_get_historical_info_with_args(self,): data = self.stock.get_historical_info('YHOO',items=['Open','Close','High','Low'], limit=5,startDate='2014-09-11',endDate='2015-02-10') logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code,200) + self.assertEqual(data.status_code,200) + + def test_get_historical_info_without_args(self,): + data = self.stock.get_historical_info('YHOO') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code,200) def test_get_options_info(self,): data = self.stock.get_options_info('YHOO') From 65a67405c23d238a739791c42860eb6fc78ad316 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 30 May 2015 03:37:21 +0200 Subject: [PATCH 393/582] fix #96: default startDate and endDate fixed --- myql/contrib/stockscraper/stockretriever.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index f14059e..5df54bc 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -5,7 +5,9 @@ from __future__ import absolute_import -import pdb +import datetime +from datetime import date, timedelta + from myql.myql import MYQL class StockRetriever(MYQL): @@ -33,6 +35,13 @@ def get_news_feed(self, symbol,format='json'): def get_historical_info(self, symbol,items=None, startDate=None, endDate=None, limit=None, format='json'): """get_historical_info() uses the csv datatable to retrieve all available historical data on a typical historical prices page """ + today = date.today() + start_date = datetime.date(day=today.day - 7,month=today.month - 1, year=today.year) + end_date = datetime.date(day=today.day - 1,month=today.month - 1, year=today.year) + + startDate = startDate if startDate else str(start_date) + endDate = endDate if endDate else str(end_date) + response = self.select('yahoo.finance.historicaldata',items,limit).where(['symbol','=',symbol],['startDate','=',startDate],['endDate','=',endDate]) return response From 125c1074fa5370c4b2f0fa4ffb60951815c411be Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 30 May 2015 03:40:24 +0200 Subject: [PATCH 394/582] trying support for pypy3 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3d0e39b..40878d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,5 +5,6 @@ python: - "3.3" - "3.4" - "pypy" + - "pypy3" install: "pip install -r requirements.txt" script: ./run_tests.sh From a62158d8af9ed1b2b049106bf940898660d5e825 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 30 May 2015 04:05:59 +0200 Subject: [PATCH 395/582] #1: Updating documentation --- README.md | 224 ++++++++--------------------------------------------- README.rst | 8 ++ 2 files changed, 39 insertions(+), 193 deletions(-) diff --git a/README.md b/README.md index ef96fec..5f8537f 100755 --- a/README.md +++ b/README.md @@ -20,16 +20,39 @@ Yahoo! Query Language Documentation and Support * Yahoo! Social APIs - http://developer.yahoo.com/social/ * Yahoo! QUery Language Console https://developer.yahoo.com/yql/console/ +### Features -Release Notes -============== +* Simple YQL Query +* Authenticated YQL Query ( OAuth ) +* StockScraper +* YQL Open Table (Classes and Metaclasses) Generator -v 1.2.2 ( in development ) +### Installation + +```shell +$ pip install myql +``` + +### Documentation + +Full Documentation is [here](http://myql.readthedocs.org/en/latest/) + +### Contribute + +* Report issue +* Star and Fork the repository +* Submit pull requests +* Above all, have fun playing with data :wink: + +#### Release Notes + +##### 1.2.2 ( in development ) ------- * **Python3** support OK [#71](https://github.com/josuebrunel/myql/issues/71) +* **PyPy/PyPy3** support OK * Fixed issue with **IN** condition in **where** clause -* Fixed issue when passing an empty list/tuple (**[]/()**) in a **where** clause -* Import of StockParser from Gurchet Rai https://github.com/gurch101/StockScraper OK [#68](https://github.com/josuebrunel/myql/issues/68) +* Fixed issue when passing an empty list/tuple (**[]/()**) in a **where** clause besides first argument +* Import of ***[StockParser](https://github.com/gurch101/StockScraper)*** from Gurchet Rai OK [#68](https://github.com/josuebrunel/myql/issues/68) * Insert, Update, Delete methods added [#67](https://github.com/josuebrunel/myql/issues/67) * Dummy *try/except* removed from main module * Fixed **Invalid OAuth Signature** when using a refreshed token [#64](https://github.com/josuebrunel/myql/issues/64) @@ -38,16 +61,16 @@ v 1.2.2 ( in development ) * Added useful functions in utils [#81](https://github.com/josuebrunel/myql/issues/81) * Default access to community tables -v 1.2.1 +##### v 1.2.1 ------ * Multiple requests while using OAuth fixed -v 1.2.0 +##### 1.2.0 ------- * OpenTable classes * Access to resources requiring authentication -v 0.5.6 +##### 0.5.6 ------------- * fetch data * access to community data @@ -58,190 +81,5 @@ v 0.5.6 * fix limit on ***select(...).where(...)*** when no limit value is passed * fix limit on ***get(...)*** -installation -============ - -```shell -$ pip install myql -``` - -how to use -========== - -```python ->>> import myql ->>> yql = myql.MYQL() ->>> yql.diagnostics = True # To turn diagnostics on -``` - -####access to community tables - -```python ->>> yql = myql.MYQL() ->>> rep = yql.rawQuery('desc yahoo.finance.quotes ') ->>> rep.json() -{u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} ->>> yql.community= True # Setting up access to community ->>> rep = yql.rawQuery('desc yahoo.finance.quotes ') ->>> rep.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} ->>> -``` - -***OR*** - -```python ->>> import myql ->>> yql = myql.MYQL(community=True) ->>> # do your magic -``` - -####changing response format (xml or json) - -The response format is by default ***json***. - -```python ->>> import myql ->>> yql = myql.MYQL(format='xml', community=True) ->>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') ->>> rep.text -u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' ->>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') ->>> rep.json() -{u'query': {u'count': 11, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'55998384', u'name': u'Cuvette-Ouest Department'}, {u'woeid': u'2344968', u'name': u'Cuvette Department'}, {u'woeid': u'2344973', u'name': u'Plateaux District'}, {u'woeid': u'2344974', u'name': u'Sangha'}, {u'woeid': u'2344970', u'name': u'Lekoumou'}, {u'woeid': u'2344975', u'name': u'Pool Department'}, {u'woeid': u'2344971', u'name': u'Likouala Department'}, {u'woeid': u'2344972', u'name': u'Niari Department'}, {u'woeid': u'2344976', u'name': u'Brazzaville'}, {u'woeid': u'2344967', u'name': u'Bouenza Department'}, {u'woeid': u'2344969', u'name': u'Kouilou'}]}, u'created': u'2014-08-27T04:52:38Z'}} ->>> -``` - -Methods -------- - -####use(yql_table_url,name=yql_table_name ) -Changes the data provider - -```python ->>> yql.use('http://www.josuebrunel.org//users.xml', name='myusers') -``` - -####desc(tablename) -Returns table description - -```python ->>> response = yql.desc('weather.forecast') ->>> response.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'request': {u'select': [{u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'location'}, {u'type': u'xs:string', u'name': u'u'}]}, {u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'woeid'}, {u'type': u'xs:string', u'name': u'u'}]}]}, u'security': u'ANY', u'meta': {u'documentationURL': u'http://developer.yahoo.com/weather/', u'sampleQuery': u'select * from weather.forecast where woeid=2502265', u'description': u'Weather forecast table', u'author': u'Yahoo! Inc'}, u'hash': u'aae78b1462a6a8fbc748aec4cf292767', u'name': u'weather.forecast'}}, u'created': u'2014-08-16T19:31:51Z'}} ->>> -``` - -####rawQuery(query) - -Allows you to directly type your query - -```python ->>> response = yql.rawQuery("select * from geo.countries where place='North America'") ->>> # deal with the response -``` - -####select(table, fields, limit).where(filters, ...) - -Select a table i.e *weather.forecast*. -If *table* not provided, it will use the default table. If there's no such thing as a default table, it will raise a *NoTableSelectedError* - -***NB*** : A simple select doesn't return any data. Use ***GET*** instead. - -```python ->>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) ->>> response.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424775', u'name': u'Canada'}}, u'created': u'2014-08-16T19:04:08Z'}} ->>> ... ->>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', '=', 'Africa']) ->>> rep.json() -{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T10:52:49Z'}} ->>> ->>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) ->>> rep.json() -{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T11:22:49Z'}} ->>> -``` - -####get(table, fields, limit) -Same as ***SELECT***, but instead returns data. - -**REMINDER** : Some tables require a **where clause**, therefore ***GET*** won't work on those tables, use *select(...).where(...)* instead . - -```python ->>> rep = yql.get('geo.countries', ['name', 'woeid'], 1) ->>> rep.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} ->>> -``` - -####insert(table, (field1, field2, ..., fieldN),(value1, value2, ..., valueN)) -Insert values into a table. Arguments 2 and 3 may be **tuples** or **list**. - -```python ->>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) ->>> response.json() # result prettyfied just for the example -{ - "query": { - "count": 1, - "created": "2015-05-14T13:25:56Z", - "lang": "en-US", - "results": { - "inserted": { - "execute": "store://KkkC5xDw4v32IcWWSQ4YRe", - "select": "store://Zc5LHXcmYM7XBfSbo9tzFL", - "update": "store://Rqb5fbQyDvrfHJiClWnZ6q" - } - } - } -} -``` - -####update(table,[field1, ..., fieldN],[value1, ..., ...valueN]).where(filters, ...) -Update fields values. This method __is always followed by ***where()***__. Arguments 2 and 3 may be **tuples** or **list**. - -```python ->>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) ->>> response.json() # result prettyfied just for the example -{ - "query": { - "count": 1, - "created": "2015-05-14T13:32:52Z", - "lang": "en-US", - "results": { - "success": "Updated store://KkkC5xDw4v32IcWWSQ4YRe" - } - } -} -``` - -####delete(table).where(filters, ...) -Delete records -```python ->>> response = self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) ->>> response.json() # result prettyfied just for the example -{ - "query": { - "count": 1, - "created": "2015-05-14T13:38:28Z", - "lang": "en-US", - "results": { - "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" - } - } -} - -``` - -Using OAuth to fetch protected resources -========================================= - -```python ->>> from myql.contrib.auth import YOAuth ->>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. ->>> from myql import MYQL ->>> yql = MYQL(format='xml', oauth=oauth) ->>> response = yql.getGUID('josue_brunel') # Deal with the response -``` diff --git a/README.rst b/README.rst index 3505082..b51099e 100644 --- a/README.rst +++ b/README.rst @@ -13,6 +13,14 @@ Yahoo! Query Language Documentation and Support - `Yahoo! Query Language Console `__ +Features +======== + +* Simple YQL Query +* Authenticated YQL Query ( OAuth ) +* StockScraper +* YQL Open Table (Classes and Metaclasses) Generator + Installation ============ From 478249f495f98b5f10ef39802ffc34daf7bd78e6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 30 May 2015 04:13:21 +0200 Subject: [PATCH 396/582] #1 : updating documentation --- README.md | 2 ++ docs/index.md | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5f8537f..19c446e 100755 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Yahoo! Query Language Documentation and Support * Authenticated YQL Query ( OAuth ) * StockScraper * YQL Open Table (Classes and Metaclasses) Generator +* Response prettyfier ### Installation @@ -60,6 +61,7 @@ Full Documentation is [here](http://myql.readthedocs.org/en/latest/) * Fixed format issue [#82](https://github.com/josuebrunel/myql/issues/82) * Added useful functions in utils [#81](https://github.com/josuebrunel/myql/issues/81) * Default access to community tables +* Response prettyfier : *pretty_json, pretty_xml* ##### v 1.2.1 ------ diff --git a/docs/index.md b/docs/index.md index 8d4f0cd..2c7f4ba 100644 --- a/docs/index.md +++ b/docs/index.md @@ -133,8 +133,9 @@ Same as ***SELECT***, but instead returns data. Insert values into a table. Arguments 2 and 3 may be **tuples** or **list**. ```python +>>> from myql.utils import pretty_json >>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) ->>> response.json() # result prettyfied just for the example +>>> print(pretty_json(response.content)) { "query": { "count": 1, @@ -155,8 +156,9 @@ Insert values into a table. Arguments 2 and 3 may be **tuples** or **list**. Update fields values. This method __is always followed by ***where()***__. Arguments 2 and 3 may be **tuples** or **list**. ```python +>>> from myql.utils import pretty_json >>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) ->>> response.json() # result prettyfied just for the example +>>> print(pretty_json(response.content)) { "query": { "count": 1, @@ -172,8 +174,9 @@ Update fields values. This method __is always followed by ***where()***__. Argum ####delete(table).where(filters, ...) Delete records ```python +>>> from myql.utils import pretty_json >>> response = self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) ->>> response.json() # result prettyfied just for the example +>>> print(pretty_json(response.content)) { "query": { "count": 1, From bab747201765469bef76997cdabd1ca5adea7821 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 31 May 2015 04:12:15 +0200 Subject: [PATCH 397/582] #1: Full Doc Link in Bold --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 19c446e..99a77a8 100755 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ mYQL [![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Issues](https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg)](https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql) -mYQL is a Python wrapper of the Yahoo Query Language. Full documentation [here](http://myql.readthedocs.org/en/latest/) +mYQL is a Python wrapper of the Yahoo Query Language. **[Full Documentation](http://myql.readthedocs.org/en/latest/)** Yahoo! Query Language Documentation and Support =============================================== From 0c5118013198b0da0530e0bee6604317717d698a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 31 May 2015 04:22:07 +0200 Subject: [PATCH 398/582] fixing classifiers --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4fabd28..0f04982 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,9 @@ def read(fname): classifiers = [ 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Development Status :: 5 - Stable', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Development Status :: 5 - Production/Stable', 'Software Development :: Libraries :: Python Modules', 'Environment :: Console', 'Intended Audience :: Developers', From 22172b17dc7669c006fc6b6faaa977dc49c4fbf9 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 31 May 2015 04:24:29 +0200 Subject: [PATCH 399/582] fix invalid classifier : Software Development :: ... --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0f04982..3d0ecdd 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def read(fname): 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Development Status :: 5 - Production/Stable', - 'Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: Libraries :: Python Modules', 'Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License' From c7527dbc9a2367bef1cf7de93f1c7d8eafdb29d5 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 31 May 2015 04:54:14 +0200 Subject: [PATCH 400/582] #1: link to doc as name of lib --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99a77a8..d02e46b 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -mYQL +[mYQL](http://myql.readthedocs.org/en/latest/) ========= [![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) From d782b85a294a56c5c12c8988b3d3d77e0ed0a7d1 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 07:25:13 +0200 Subject: [PATCH 401/582] #1 : examples added --- README.md | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d02e46b..660379b 100755 --- a/README.md +++ b/README.md @@ -34,6 +34,103 @@ Yahoo! Query Language Documentation and Support $ pip install myql ``` +#### Examples +-------------- + +* ___rawQuery___ + +```python +>>> import myql +>>> from myql.utils import pretty_json, pretty_xml +>>> yql = myql.MYQL(format='json') +>>> response = yql.rawQuery("select name, woeid from geo.states where place='Congo' limit 2") +>>> print(pretty_json(response.content)) +{ + "query": { + "count": 2, + "created": "2015-06-03T04:51:23Z", + "lang": "en-US", + "results": { + "place": [ + { + "name": "Cuvette-Ouest Department", + "woeid": "55998384" + }, + { + "name": "Cuvette Department", + "woeid": "2344968" + } + ] + } + } +} +>>> + +``` + +* ___get(tabke, item=[], limit=None)___ + +```python +>>> response = yql.get('geo.countries',['name,woeid'],3) +>>> print(pretty_json(response.content)) +{ + "query": { + "count": 3, + "created": "2015-06-03T05:07:47Z", + "lang": "en-US", + "results": { + "place": [ + { + "name": "Sao Tome and Principe", + "woeid": "23424966" + }, + { + "name": "Ghana", + "woeid": "23424824" + }, + { + "name": "Togo", + "woeid": "23424965" + } + ] + } + } +} +>>> + +``` + +* ___select(table,item=[],limit=None).were(condition)___ + +```python +>>> yql.format = 'xml' +>>> response = yql.select('weather.forecast',['units','atmosphere']).where(['woeid','in','select woeid from geo.places(1) where text="Paris,Fr"']) +>>> print(pretty_xml(response.content)) + + + + + + + + + + + + + + + + + + + + + + + +``` + ### Documentation Full Documentation is [here](http://myql.readthedocs.org/en/latest/) @@ -47,7 +144,7 @@ Full Documentation is [here](http://myql.readthedocs.org/en/latest/) #### Release Notes -##### 1.2.2 ( in development ) +##### 1.2.2 ------- * **Python3** support OK [#71](https://github.com/josuebrunel/myql/issues/71) * **PyPy/PyPy3** support OK From bdbb4f6d19977aa21902e9bd3628b2820d3ae40f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 07:27:21 +0200 Subject: [PATCH 402/582] #1 :updating documentation --- docs/index.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/index.md b/docs/index.md index 2c7f4ba..f4f8bf9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,24 +28,25 @@ Quick Start >>> yql.diagnostics = True # To turn diagnostics on ``` -####Access to community tables +####Disable access to community tables ```python ->>> yql = myql.MYQL() >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') >>> rep.json() -{u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} +{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} +>>> >>> yql.community= True # Setting up access to community +>>> yql = myql.MYQL() >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') >>> rep.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} ->>> +{u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} + ``` or ```python >>> import myql ->>> yql = myql.MYQL(community=True) +>>> yql = myql.MYQL(community=False) >>> # do your magic ``` From 7bcec3162a6517fc230915395f7e82f2cb5a4262 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 09:17:41 +0200 Subject: [PATCH 403/582] #106: IN Condition test added --- tests/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 23004e9..8fd71f9 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -88,6 +88,14 @@ def test_select_in(self,): logging.error(e) self.assertEqual(response.status_code,200) + def test_select_in_2(self,): + response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','in',("select woeid from geo.places(1) where text='Paris,Fr'")]) + try: + logging.debug(pretty_json(response.content)) + except (Exception,) as e: + logging.error(e) + self.assertEqual(response.status_code,200) + def test_1_insert(self,): response = self.yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) try: From 58126b2e4cea72ccab89c0cb81efd2b5d1934b7f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 09:23:09 +0200 Subject: [PATCH 404/582] fix #105: raw_input not defined in Python3 --- myql/contrib/auth/yoauth.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index c8d83b2..7d9a011 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -1,6 +1,12 @@ """ YOAuth is inspired from Darren Kempiners YahooAPI https://github.com/dkempiners/python-yahooapi/blob/master/yahooapi.py """ +try: + input = raw_input +except (NameError,): + pass + + from __future__ import absolute_import import json @@ -66,7 +72,7 @@ def __init__(self, consumer_key, consumer_secret, **kwargs): authorize_url = AUTHORIZE_TOKEN_URL+request_token logging.debug(authorize_url) webbrowser.open(authorize_url) - verifier = raw_input("Enter verifier : ") + verifier = input("Enter verifier : ") logging.debug("VERIFIER = {0}".format(verifier)) self.token_time = time.time() From cbf59ca74a5a00e00e173d8cd24cad417029b9e6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 09:25:29 +0200 Subject: [PATCH 405/582] #106: fix from __future__ imports must be at the begining of the file --- myql/contrib/auth/yoauth.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py index 7d9a011..990ccbb 100644 --- a/myql/contrib/auth/yoauth.py +++ b/myql/contrib/auth/yoauth.py @@ -1,11 +1,6 @@ """ YOAuth is inspired from Darren Kempiners YahooAPI https://github.com/dkempiners/python-yahooapi/blob/master/yahooapi.py """ -try: - input = raw_input -except (NameError,): - pass - from __future__ import absolute_import @@ -19,6 +14,12 @@ from myql.utils import json_write_data, json_get_data +try: + input = raw_input +except (NameError,): + pass + + BASE_URL = "http://query.yahooapis.com/v1/yql" REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token" ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token" From f988ab378a0f5c6e0aa30607d272b85769e92eb2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 09:28:50 +0200 Subject: [PATCH 406/582] fix #106: fix IN condition one more time :facepalm: --- myql/myql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 6efdc6c..f380b8b 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -106,7 +106,8 @@ def clauseFormatter(self, cond): args is a list of ['column', 'operator', 'value'] ''' if cond[1].lower() == 'in': - cond[2] = "({0})".format(','.join(map(str,[ "'{0}'".format(e) for e in cond[2] ]))) + #cond[2] = "({0})".format(','.join(map(str,[ "'{0}'".format(e) for e in cond[2] ]))) + cond[2] = "{0}".format(str(cond[2])) cond = ' '.join(cond) else: cond[2] = "'{0}'".format(cond[2]) From 3a8bb4fae08d9fe5370151a46c8c4d24d533e786 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 09:29:19 +0200 Subject: [PATCH 407/582] #106 test updated --- tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 8fd71f9..363abd0 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -89,7 +89,7 @@ def test_select_in(self,): self.assertEqual(response.status_code,200) def test_select_in_2(self,): - response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','in',("select woeid from geo.places(1) where text='Paris,Fr'")]) + response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','in',("select woeid from geo.places(1) where text='Paris,Fr'"),]) try: logging.debug(pretty_json(response.content)) except (Exception,) as e: From dd9913cc15a47e305d2b2a77788f4343883ecda1 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 10:11:08 +0200 Subject: [PATCH 408/582] fix #106: IN condition fixed --- myql/myql.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index f380b8b..e18e6b0 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -106,10 +106,12 @@ def clauseFormatter(self, cond): args is a list of ['column', 'operator', 'value'] ''' if cond[1].lower() == 'in': - #cond[2] = "({0})".format(','.join(map(str,[ "'{0}'".format(e) for e in cond[2] ]))) - cond[2] = "{0}".format(str(cond[2])) - cond = ' '.join(cond) - else: + if len(cond[2]) > 1: + cond[2] = "('{0}')".format(','.join(map(str,[ "{0}".format(e) for e in cond[2] ]))) + else: + cond[2] = "({0})".format(','.join(map(str,[ "{0}".format(e) for e in cond[2] ]))) + cond = " ".join(cond) + else: cond[2] = "'{0}'".format(cond[2]) cond = ''.join(cond) From 5e3d5aca5991205484a7dcf4d8acc53f83d0aa78 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 10:14:21 +0200 Subject: [PATCH 409/582] closes #106 : fix IN condition --- README.md | 2 +- tests/tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 660379b..74d4189 100755 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ $ pip install myql ```python >>> yql.format = 'xml' ->>> response = yql.select('weather.forecast',['units','atmosphere']).where(['woeid','in','select woeid from geo.places(1) where text="Paris,Fr"']) +>>> response = yql.select('weather.forecast',['units','atmosphere']).where(['woeid','in',('select woeid from geo.places(1) where text="Paris,Fr"',)]) # IN requires tuple >>> print(pretty_xml(response.content)) diff --git a/tests/tests.py b/tests/tests.py index 363abd0..baa4a66 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -89,7 +89,7 @@ def test_select_in(self,): self.assertEqual(response.status_code,200) def test_select_in_2(self,): - response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','in',("select woeid from geo.places(1) where text='Paris,Fr'"),]) + response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','IN',('select woeid from geo.places(1) where text="Paris"',)]) try: logging.debug(pretty_json(response.content)) except (Exception,) as e: From b80370a35abbee8ee77528cca49cf3583a6d65ff Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 10:53:28 +0200 Subject: [PATCH 410/582] fix #107: using timedelta --- myql/contrib/stockscraper/stockretriever.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index 5df54bc..7f55edc 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -5,6 +5,7 @@ from __future__ import absolute_import +import calendar import datetime from datetime import date, timedelta @@ -36,8 +37,8 @@ def get_historical_info(self, symbol,items=None, startDate=None, endDate=None, l """get_historical_info() uses the csv datatable to retrieve all available historical data on a typical historical prices page """ today = date.today() - start_date = datetime.date(day=today.day - 7,month=today.month - 1, year=today.year) - end_date = datetime.date(day=today.day - 1,month=today.month - 1, year=today.year) + start_date = today - timedelta(days=30) + end_date = start_date + timedelta(days=7) startDate = startDate if startDate else str(start_date) endDate = endDate if endDate else str(end_date) From eafdc917d7062c18191412ad9e351cd56b8d155d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 11:44:38 +0200 Subject: [PATCH 411/582] closes #107: returning last week data by default --- myql/contrib/stockscraper/stockretriever.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/stockscraper/stockretriever.py index 7f55edc..fdf8895 100644 --- a/myql/contrib/stockscraper/stockretriever.py +++ b/myql/contrib/stockscraper/stockretriever.py @@ -37,8 +37,8 @@ def get_historical_info(self, symbol,items=None, startDate=None, endDate=None, l """get_historical_info() uses the csv datatable to retrieve all available historical data on a typical historical prices page """ today = date.today() - start_date = today - timedelta(days=30) - end_date = start_date + timedelta(days=7) + start_date = today - timedelta(days=today.weekday(), weeks=1) + end_date = start_date + timedelta(days=4) startDate = startDate if startDate else str(start_date) endDate = endDate if endDate else str(end_date) From 8249a3290042f26f16027cf7a94402106e2f078b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 14:07:08 +0200 Subject: [PATCH 412/582] #1: updating documentation --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 74d4189..c7c55a8 100755 --- a/README.md +++ b/README.md @@ -131,6 +131,12 @@ $ pip install myql ``` +* ___Using OAUTH___ + +```python + +``` + ### Documentation Full Documentation is [here](http://myql.readthedocs.org/en/latest/) @@ -144,6 +150,13 @@ Full Documentation is [here](http://myql.readthedocs.org/en/latest/) #### Release Notes +##### 1.2.3 +------- + +* Fixed issue related to date in StockRetriver.get_historical_info [#107](https://github.com/josuebrunel/myql/issues/107) +* Fixed issue with **IN** condition in **where** clause [#106](https://github.com/josuebrunel/myql/issues/107) +* Fix definition of raw_input for python3 [#105](https://github.com/josuebrunel/myql/issues/105) + ##### 1.2.2 ------- * **Python3** support OK [#71](https://github.com/josuebrunel/myql/issues/71) From 777396a895b3d07fc5b29119494f36573d3c030b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 3 Jun 2015 20:34:47 +0200 Subject: [PATCH 413/582] development packages added --- dev_requirements.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 dev_requirements.txt diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..9f0954a --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1,16 @@ +Jinja2==2.7.3 +Markdown==2.6.2 +MarkupSafe==0.23 +PyYAML==3.11 +backports.ssl-match-hostname==3.4.0.2 +certifi==2015.04.28 +click==4.0 +ghp-import==0.4.1 +livereload==2.4.0 +mkdocs==0.13.3 +oauthlib==0.7.2 +rauth==0.7.1 +requests==2.7.0 +requests-oauthlib==0.5.0 +six==1.9.0 +tornado==4.2 From a2717daa93ff4eec1362f7d4acb581b52a87d00c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 02:32:20 +0200 Subject: [PATCH 414/582] securing credentials --- .travis.yml | 19 +++++++++++-------- run_tests.sh | 7 +------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40878d1..17d723e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,13 @@ language: python python: - - "2.7" - - "3.2" - - "3.3" - - "3.4" - - "pypy" - - "pypy3" -install: "pip install -r requirements.txt" -script: ./run_tests.sh +- '2.7' +- '3.2' +- '3.3' +- '3.4' +- pypy +- pypy3 +install: pip install -r requirements.txt +script: "./run_tests.sh" +env: + global: + secure: hXpkZqclyUXMX586jS4BtJmYsrszVr/jWbMiLWLOZ2z7n5LK9INziKkOqYu0JTBskAHVEmhxJ7oNOD9gI/06btLO5NLBONN3qtsHq4UruCo0Zlx2BgaDfR5FqnrVqk+fMbLCbzWgo0wShM4o8jTGy7l22xqhhYAsuubmayqEfPk= diff --git a/run_tests.sh b/run_tests.sh index 82ebe3d..000906d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,9 +1,4 @@ -set +x -rm -rf credentials.json -wget `echo 'U2FsdGVkX19Wg/Os0JMcl3kXdAaNcgSF+fAg4oCz5zUIrCQyX3FwXeaqOAaj8YGT -GrYMpNIsovfk6uB+ZbHBjg==' | openssl enc -aes-128-cbc -a -d -salt -pass pass:url` - -set -x +wget $credentials if [ ! -z $1 ]; then TestCase=".${1}" From d2a18bc6dd4b8d6e31c3262b180088ee31154017 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 03:29:27 +0200 Subject: [PATCH 415/582] encrypted file added --- credentials.json.enc | Bin 0 -> 1232 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 credentials.json.enc diff --git a/credentials.json.enc b/credentials.json.enc new file mode 100644 index 0000000000000000000000000000000000000000..062ed7f5effb5ad6ade268c340438d73273be9d2 GIT binary patch literal 1232 zcmV;>1TXu^&vO?I;@t6d$lI`96W#1a1pQS+Gtd>gPSALwuMB!@2a_j9dAJ9jT-kXn zw0#=s{32e%ieI}~Kb5A0{foAh=nSaJ;`>th_Cv&%>j?0-7{PWf*8`$0U^r*^%hS5`4w zZIkm|s{V6=&n*KK>nQyQ2^r{~CR<%d{|8bGtIS%!96Krd8inwJfBMU=MsI;xzOb^N zhd148X+B@D91Zpi3fib6#;^%azh&rv`E7{|KNl{o$O||w9Z_9aTPVr}eV(5+L zg3zHhzsBr0R-!|mQENKrnA5zINiZRne>}Kz%DH?RYnj)sU=R(njMzqv43mIbB^#~{ z&rkv7jD~4>d>*;zs9SKfIURHz34ij!B~KiAW|UV%hB@Lxh5`bdbMZ(dn|rY+z6@iq zgXC~TQGAn?7@+dULYyeLlIeu}1h#>4xkYV)=ts0yO!1|_`C@STmndR30Vur#58wrQ z(7<@pC)La}^7^l~Q7dE~&BLY{+ThNdSOaAp$+aX8(+69GgU`?xj5VIFLJ;)u~aK-u6osqr{FTOs39bVLXj4Nu5nm^ zASdy}HU-72X?IpR zJl@{Tg^U+mou>5*I^K5MYcA(hXl|cW=KqIh+%9W_O%)sRM>O^KX0L3GN`HF`lQsXk zu@Sb@(+ujniz(zfZ3lP9N1pfirGfPQ;V0)L=^gWd)H>n`&x)lovB)BvW_~=1JmVqL z=VgfR`a5r=y%70m3b{_cuv-^F5n{j{S~scMF{VAmumqe&fJN|HZ2qG9;SYl7k%=IC zgBfXD_i|1VSFX`WZhHbCP%*6-n9IdD`j~H51#)+L zlJ90zM&%eOeC_(3S!FhAk_?H43HlQE`)Al18|utGtcRzlmkdNtFppxWEi93*#QOIw zqBbd!fs~Yg{`V>@;Gt9{HYIDDfX)jFs&z0Izn^#n8>a2Kh6%lCtDXK>jmRNljvL|? z&m;YenCv@g2y8{VhKfmdvGYH=CLdRjC9GjPC8Ev>7#a_Id``zb7?2&em;#lHQ8+5; zu%oM2VsPKe)X8%u_296qMC=>)i1pUxl$>k-0vq9rMfIg3e*bv1Ui=SwSlIBl-Sa^5!(QcuV-oYRJ)GwwO#gy zt+g5zG<2`$z~4C=H<9W9rRsHKq*T4^emHT`1wx9Ek`+^N*9!T!y=(`RkiT`a5HmoB u4JQ8cDIsm(dk^D&wR$lfJi?C2+nsTzaf&Gs Date: Sat, 6 Jun 2015 03:29:58 +0200 Subject: [PATCH 416/582] updating travis config --- .travis.yml | 3 +++ run_tests.sh | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17d723e..8d090c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,3 +11,6 @@ script: "./run_tests.sh" env: global: secure: hXpkZqclyUXMX586jS4BtJmYsrszVr/jWbMiLWLOZ2z7n5LK9INziKkOqYu0JTBskAHVEmhxJ7oNOD9gI/06btLO5NLBONN3qtsHq4UruCo0Zlx2BgaDfR5FqnrVqk+fMbLCbzWgo0wShM4o8jTGy7l22xqhhYAsuubmayqEfPk= +before_install: +- openssl aes-256-cbc -K $encrypted_60f42691f4f2_key -iv $encrypted_60f42691f4f2_iv + -in credentials.json.enc -out credentials.json -d diff --git a/run_tests.sh b/run_tests.sh index 000906d..65fd2f4 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,5 +1,3 @@ -wget $credentials - if [ ! -z $1 ]; then TestCase=".${1}" else From 61e8192bd875fe08925ea23c892dd54d5cae51b4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 03:30:23 +0200 Subject: [PATCH 417/582] weird updated --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c7c55a8..6847e48 100755 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ $ pip install myql ``` -* ___get(tabke, item=[], limit=None)___ +* ___get(table, item=[], limit=None)___ ```python >>> response = yql.get('geo.countries',['name,woeid'],3) From bfcbea2692862cb4749e3dd91f4222fa20cfc6af Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 16:51:56 +0200 Subject: [PATCH 418/582] #1: updating documentation --- README.md | 251 +++++++++++++++++++++++++++++++++++------------------ README.rst | 178 ++++++++++++++++++++++++++++--------- 2 files changed, 305 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 6847e48..2209fce 100755 --- a/README.md +++ b/README.md @@ -28,125 +28,210 @@ Yahoo! Query Language Documentation and Support * YQL Open Table (Classes and Metaclasses) Generator * Response prettyfier -### Installation +mYQL +========= + +mYQL is a Python wrapper of the Yahoo Query Language. + +Yahoo! Query Language Documentation and Support +=============================================== + +* [Yahoo! Query Language](http://developer.yahoo.com/yql/) +* [Yahoo! Developer Network](http://developer.yahoo.com) +* [Yahoo! Application Platform](http://developer.yahoo.com/yap/) +* [Yahoo! Social APIs](http://developer.yahoo.com/social/) +* [Yahoo! Query Language Console](https://developer.yahoo.com/yql/console/) + +Installation +============ ```shell $ pip install myql ``` -#### Examples --------------- +Quick Start +=========== + +```python +>>> import myql +>>> yql = myql.MYQL() +>>> yql.diagnostics = True # To turn diagnostics on +``` + +####Disable access to community tables -* ___rawQuery___ +```python +>>> rep = yql.rawQuery('desc yahoo.finance.quotes ') +>>> rep.json() +{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} +>>> +>>> yql.community= True # Setting up access to community +>>> yql = myql.MYQL() +>>> rep = yql.rawQuery('desc yahoo.finance.quotes ') +>>> rep.json() +{u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} + +``` +or ```python >>> import myql ->>> from myql.utils import pretty_json, pretty_xml ->>> yql = myql.MYQL(format='json') ->>> response = yql.rawQuery("select name, woeid from geo.states where place='Congo' limit 2") +>>> yql = myql.MYQL(community=False) +>>> # do your magic +``` + +####Changing response format (xml or json) + +The response format is by default ***json***. + +```python +>>> import myql +>>> yql = myql.MYQL(format='xml', community=True) +>>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') +>>> rep.text +u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' +>>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') +>>> rep.json() +{u'query': {u'count': 11, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'55998384', u'name': u'Cuvette-Ouest Department'}, {u'woeid': u'2344968', u'name': u'Cuvette Department'}, {u'woeid': u'2344973', u'name': u'Plateaux District'}, {u'woeid': u'2344974', u'name': u'Sangha'}, {u'woeid': u'2344970', u'name': u'Lekoumou'}, {u'woeid': u'2344975', u'name': u'Pool Department'}, {u'woeid': u'2344971', u'name': u'Likouala Department'}, {u'woeid': u'2344972', u'name': u'Niari Department'}, {u'woeid': u'2344976', u'name': u'Brazzaville'}, {u'woeid': u'2344967', u'name': u'Bouenza Department'}, {u'woeid': u'2344969', u'name': u'Kouilou'}]}, u'created': u'2014-08-27T04:52:38Z'}} +>>> +``` + + +Methods +------- + +####use(yql_table_url,name=yql_table_name) +Changes the data provider + +```python +>>> yql.use('http://www.josuebrunel.org//users.xml', name='myusers') +``` + +####desc(tablename) +Returns table description + +```python +>>> response = yql.desc('weather.forecast') +>>> response.json() +{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'request': {u'select': [{u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'location'}, {u'type': u'xs:string', u'name': u'u'}]}, {u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'woeid'}, {u'type': u'xs:string', u'name': u'u'}]}]}, u'security': u'ANY', u'meta': {u'documentationURL': u'http://developer.yahoo.com/weather/', u'sampleQuery': u'select * from weather.forecast where woeid=2502265', u'description': u'Weather forecast table', u'author': u'Yahoo! Inc'}, u'hash': u'aae78b1462a6a8fbc748aec4cf292767', u'name': u'weather.forecast'}}, u'created': u'2014-08-16T19:31:51Z'}} +>>> +``` + +####rawQuery(query) + +Allows you to directly type your query + +```python +>>> response = yql.rawQuery("select * from geo.countries where place='North America'") +>>> # deal with the response +``` + +####select(table, fields, limit).where(filters, ...) + +Select a table i.e *weather.forecast*. +If *table* not provided, it will use the default table. If there's no such thing as a default table, it will raise a *NoTableSelectedError* + +***NB*** : A simple select doesn't return any data. Use ***GET*** instead. + +```python +>>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) +>>> response.json() +{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424775', u'name': u'Canada'}}, u'created': u'2014-08-16T19:04:08Z'}} +>>> ... +>>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', '=', 'Africa']) +>>> rep.json() +{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T10:52:49Z'}} +>>> +>>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) +>>> rep.json() +{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T11:22:49Z'}} +>>> +``` + +####get(table, fields, limit) +Same as ***SELECT***, but instead returns data. + +**REMINDER** : Some tables require a **where clause**, therefore ***GET*** won't work on those tables, use *select(...).where(...)* instead . + +```python +>>> yql.get('geo.countries', ['name', 'woeid'], 1) +>>> rep.json() +{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} +>>> +``` + +####insert(table, (field1, field2, ..., fieldN),(value1, value2, ..., valueN)) +Insert values into a table. Arguments 2 and 3 may be **tuples** or **list**. + +```python +>>> from myql.utils import pretty_json +>>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) >>> print(pretty_json(response.content)) { "query": { - "count": 2, - "created": "2015-06-03T04:51:23Z", - "lang": "en-US", + "count": 1, + "created": "2015-05-14T13:25:56Z", + "lang": "en-US", "results": { - "place": [ - { - "name": "Cuvette-Ouest Department", - "woeid": "55998384" - }, - { - "name": "Cuvette Department", - "woeid": "2344968" - } - ] + "inserted": { + "execute": "store://KkkC5xDw4v32IcWWSQ4YRe", + "select": "store://Zc5LHXcmYM7XBfSbo9tzFL", + "update": "store://Rqb5fbQyDvrfHJiClWnZ6q" + } } } } ->>> - ``` -* ___get(table, item=[], limit=None)___ +####update(table,[field1, ..., fieldN],[value1, ..., ...valueN]).where(filters, ...) +Update fields values. This method __is always followed by ***where()***__. Arguments 2 and 3 may be **tuples** or **list**. ```python ->>> response = yql.get('geo.countries',['name,woeid'],3) +>>> from myql.utils import pretty_json +>>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) >>> print(pretty_json(response.content)) { "query": { - "count": 3, - "created": "2015-06-03T05:07:47Z", - "lang": "en-US", + "count": 1, + "created": "2015-05-14T13:32:52Z", + "lang": "en-US", "results": { - "place": [ - { - "name": "Sao Tome and Principe", - "woeid": "23424966" - }, - { - "name": "Ghana", - "woeid": "23424824" - }, - { - "name": "Togo", - "woeid": "23424965" - } - ] + "success": "Updated store://KkkC5xDw4v32IcWWSQ4YRe" } } } ->>> - ``` -* ___select(table,item=[],limit=None).were(condition)___ - +####delete(table).where(filters, ...) +Delete records ```python ->>> yql.format = 'xml' ->>> response = yql.select('weather.forecast',['units','atmosphere']).where(['woeid','in',('select woeid from geo.places(1) where text="Paris,Fr"',)]) # IN requires tuple ->>> print(pretty_xml(response.content)) - - - - - - - - - - - - - - - - - - - - - - - +>>> from myql.utils import pretty_json +>>> response = self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) +>>> print(pretty_json(response.content)) +{ + "query": { + "count": 1, + "created": "2015-05-14T13:38:28Z", + "lang": "en-US", + "results": { + "success": "store://Rqb5fbQyDvrfHJiClWnZ6q deleted" + } + } +} + ``` -* ___Using OAUTH___ +Using OAuth to fetch protected resources +========================================= ```python - +>>> from myql.contrib.auth import YOAuth +>>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. +>>> from myql import MYQL +>>> yql = MYQL(format='xml', oauth=oauth) +>>> response = yql.getGUID('josue_brunel') # Deal with the response ``` -### Documentation - -Full Documentation is [here](http://myql.readthedocs.org/en/latest/) - -### Contribute - -* Report issue -* Star and Fork the repository -* Submit pull requests -* Above all, have fun playing with data :wink: #### Release Notes diff --git a/README.rst b/README.rst index b51099e..fd727b2 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,36 @@ +`mYQL `__ +================================================= + +|Build Status| |Documentation Status| |Latest Version| |Downloads| +|Py\_Versions| |Implementations| |Join the chat at +https://gitter.im/josuebrunel/myql| |Code Issues| + +mYQL is a Python wrapper of the Yahoo Query Language. **`Full +Documentation `__** + +Yahoo! Query Language Documentation and Support +=============================================== + +- Yahoo! Query Language - http://developer.yahoo.com/yql/ +- Yahoo! Developer Network: http://developer.yahoo.com +- Yahoo! Application Platform - http://developer.yahoo.com/yap/ +- Yahoo! Social APIs - http://developer.yahoo.com/social/ +- Yahoo! QUery Language Console + https://developer.yahoo.com/yql/console/ + +Features +~~~~~~~~ + +- Simple YQL Query +- Authenticated YQL Query ( OAuth ) +- StockScraper +- YQL Open Table (Classes and Metaclasses) Generator +- Response prettyfier + mYQL ==== -mYQL is a Python wrapper of the Yahoo Query Language. Read the full Documentation `http://myql.readthedocs.org/en/latest/` +mYQL is a Python wrapper of the Yahoo Query Language. Yahoo! Query Language Documentation and Support =============================================== @@ -13,51 +42,43 @@ Yahoo! Query Language Documentation and Support - `Yahoo! Query Language Console `__ -Features -======== - -* Simple YQL Query -* Authenticated YQL Query ( OAuth ) -* StockScraper -* YQL Open Table (Classes and Metaclasses) Generator - Installation ============ -:: +.. code:: shell $ pip install myql Quick Start =========== -:: +.. code:: python >>> import myql >>> yql = myql.MYQL() >>> yql.diagnostics = True # To turn diagnostics on -Access to community tables -^^^^^^^^^^^^^^^^^^^^^^^^^^ +Disable access to community tables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:: +.. code:: python - >>> yql = myql.MYQL() >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') >>> rep.json() - {u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} + {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} + >>> >>> yql.community= True # Setting up access to community + >>> yql = myql.MYQL() >>> rep = yql.rawQuery('desc yahoo.finance.quotes ') >>> rep.json() - {u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} - >>> + {u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} or -:: +.. code:: python >>> import myql - >>> yql = myql.MYQL(community=True) + >>> yql = myql.MYQL(community=False) >>> # do your magic Changing response format (xml or json) @@ -65,7 +86,7 @@ Changing response format (xml or json) The response format is by default ***json***. -:: +.. code:: python >>> import myql >>> yql = myql.MYQL(format='xml', community=True) @@ -85,7 +106,7 @@ use(yql\_table\_url,name=yql\_table\_name) Changes the data provider -:: +.. code:: python >>> yql.use('http://www.josuebrunel.org//users.xml', name='myusers') @@ -94,7 +115,7 @@ desc(tablename) Returns table description -:: +.. code:: python >>> response = yql.desc('weather.forecast') >>> response.json() @@ -106,7 +127,7 @@ rawQuery(query) Allows you to directly type your query -:: +.. code:: python >>> response = yql.rawQuery("select * from geo.countries where place='North America'") >>> # deal with the response @@ -121,7 +142,7 @@ will raise a *NoTableSelectedError* ***NB*** : A simple select doesn't return any data. Use ***GET*** instead. -:: +.. code:: python >>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) >>> response.json() @@ -145,7 +166,7 @@ Same as ***SELECT***, but instead returns data. ***GET*** won't work on those tables, use *select(...).where(...)* instead . -:: +.. code:: python >>> yql.get('geo.countries', ['name', 'woeid'], 1) >>> rep.json() @@ -158,10 +179,11 @@ insert(table, (field1, field2, ..., fieldN),(value1, value2, ..., valueN)) Insert values into a table. Arguments 2 and 3 may be **tuples** or **list**. -:: +.. code:: python + >>> from myql.utils import pretty_json >>> response = yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) - >>> response.json() # result prettyfied just for the example + >>> print(pretty_json(response.content)) { "query": { "count": 1, @@ -183,10 +205,11 @@ update(table,[field1, ..., fieldN],[value1, ..., ...valueN]).where(filters, ...) Update fields values. This method **is always followed by ***where()*****. Arguments 2 and 3 may be **tuples** or **list**. -:: +.. code:: python + >>> from myql.utils import pretty_json >>> response = yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) - >>> response.json() # result prettyfied just for the example + >>> print(pretty_json(response.content)) { "query": { "count": 1, @@ -203,10 +226,11 @@ delete(table).where(filters, ...) Delete records -:: +.. code:: python + >>> from myql.utils import pretty_json >>> response = self.yql.delete('yql.storage').where(['name','=','store://Rqb5fbQyDvrfHJiClWnZ6q']) - >>> response.json() # result prettyfied just for the example + >>> print(pretty_json(response.content)) { "query": { "count": 1, @@ -221,7 +245,7 @@ Delete records Using OAuth to fetch protected resources ======================================== -:: +.. code:: python >>> from myql.contrib.auth import YOAuth >>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. @@ -229,12 +253,84 @@ Using OAuth to fetch protected resources >>> yql = MYQL(format='xml', oauth=oauth) >>> response = yql.getGUID('josue_brunel') # Deal with the response -` `__\ Next - --------------- - -Built with `MkDocs `__ using a -`theme `__ provided by `Read -the Docs `__. +Release Notes +^^^^^^^^^^^^^ + +##### 1.2.3 +----------- + +- Fixed issue related to date in StockRetriver.get\_historical\_info + `#107 `__ +- Fixed issue with **IN** condition in **where** clause + `#106 `__ +- Fix definition of raw\_input for python3 + `#105 `__ + +##### 1.2.2 +----------- + +- **Python3** support OK + `#71 `__ +- **PyPy/PyPy3** support OK +- Fixed issue with **IN** condition in **where** clause +- Fixed issue when passing an empty list/tuple (**[]/()**) in a + **where** clause besides first argument +- Import of + ***`StockParser `__*** from + Gurchet Rai OK + `#68 `__ +- Insert, Update, Delete methods added + `#67 `__ +- Dummy *try/except* removed from main module +- Fixed **Invalid OAuth Signature** when using a refreshed token + `#64 `__ +- Fixed misused of ***MYQL.use(...)*** + `#76 `__ +- Fixed format issue + `#82 `__ +- Added useful functions in utils + `#81 `__ +- Default access to community tables +- Response prettyfier : *pretty\_json, pretty\_xml* + +##### v 1.2.1 +------------- + +- Multiple requests while using OAuth fixed + +##### 1.2.0 +----------- + +- OpenTable classes +- Access to resources requiring authentication + +##### 0.5.6 +----------- + +- fetch data +- access to community data +- select data format (xml/json) +- change data source +- filter data +- fix handling of default response format on the fly +- fix limit on ***select(...).where(...)*** when no limit value is + passed +- fix limit on ***get(...)*** + +.. |Build Status| image:: https://travis-ci.org/josuebrunel/myql.svg?branch=master + :target: https://travis-ci.org/josuebrunel/myql +.. |Documentation Status| image:: https://readthedocs.org/projects/myql/badge/?version=latest + :target: https://myql.readthedocs.org +.. |Latest Version| image:: https://pypip.in/version/myql/badge.svg + :target: https://pypi.python.org/pypi/myql/ +.. |Downloads| image:: https://pypip.in/download/myql/badge.svg + :target: https://pypi.python.org/pypi/myql +.. |Py\_Versions| image:: https://pypip.in/py_versions/myql/badge.svg + :target: https://pypi.python.org/pypi/myql +.. |Implementations| image:: https://pypip.in/implementation/myql/badge.svg + :target: https://pypi.python.org/pypi/myql +.. |Join the chat at https://gitter.im/josuebrunel/myql| image:: https://badges.gitter.im/Join%20Chat.svg + :target: https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +.. |Code Issues| image:: https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg + :target: https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql -GitHub `« Previous <>`__ `Next » `__ From ffb61381555e7a623726808e39b89010cfa26061 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 16:54:42 +0200 Subject: [PATCH 419/582] upgrading version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3d0ecdd..9fed9af 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ __author__ = 'Josue Kouka' __email__ = 'josuebrunel@gmail.com' -__version__ = "1.2.2" +__version__ = "1.2.3" #requirements.txt with open('requirements.txt') as f: From c9cb1cfb8afb748fb30900184b03a26b0a0033c7 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 16:56:55 +0200 Subject: [PATCH 420/582] yahoo-oauth added --- dev_requirements.txt | 2 ++ requirements.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/dev_requirements.txt b/dev_requirements.txt index 9f0954a..a4e6cc2 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -14,3 +14,5 @@ requests==2.7.0 requests-oauthlib==0.5.0 six==1.9.0 tornado==4.2 +wsgiref==0.1.2 +yahoo-oauth==0.1.2 diff --git a/requirements.txt b/requirements.txt index 573ac30..df48804 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ rauth>=0.7.1 requests-oauthlib>=0.4.2 requests>=2.7.0 six>=1.9.0 +yahoo-oauth>=0.1.2 From 67fbf18630edb5fab39ba1e0e69414b57375ea82 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 18:07:35 +0200 Subject: [PATCH 421/582] updating requirements --- dev_requirements.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index a4e6cc2..f1a29c2 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -15,4 +15,4 @@ requests-oauthlib==0.5.0 six==1.9.0 tornado==4.2 wsgiref==0.1.2 -yahoo-oauth==0.1.2 +yahoo-oauth==0.1.3 diff --git a/requirements.txt b/requirements.txt index df48804..8331870 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ rauth>=0.7.1 requests-oauthlib>=0.4.2 requests>=2.7.0 six>=1.9.0 -yahoo-oauth>=0.1.2 +yahoo-oauth>=0.1.3 From 2ed1b3d09ee20736b864ce7163530e3c82dca682 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 18:12:10 +0200 Subject: [PATCH 422/582] #112: tests OK --- tests/tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index baa4a66..50305fc 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -6,9 +6,10 @@ from xml.dom import minidom from xml.etree import cElementTree as xtree +from yahoo_oauth import OAuth1 + from myql import MYQL from myql.utils import pretty_xml, pretty_json -from myql.contrib.auth import YOAuth from myql.contrib.table import Table from myql.contrib.table import Base, BaseInput @@ -153,14 +154,14 @@ def tearUp(self,): pass def test_get_guid(self,): - oauth = YOAuth(None, None, from_file='credentials.json') + oauth = OAuth1(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) response = yql.getGUID('josue_brunel') logging.debug(response.content) self.assertEqual(response.status_code,200) def test_yahoo_fantasy_sport(self,): - oauth = YOAuth(None, None, from_file='credentials.json') + oauth = OAuth1(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) teams = ('mlb.l.1328.t.1','mlb.l.1328.t.2') year = '2015-05-05' From ecb1e5e5d91ce61f002086b4804350f1243faaf4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 6 Jun 2015 18:20:15 +0200 Subject: [PATCH 423/582] fix #112: yahoo_oauth as main OAuth library --- README.md | 9 +++++---- myql/myql.py | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2209fce..51211a1 100755 --- a/README.md +++ b/README.md @@ -221,12 +221,13 @@ Delete records ``` -Using OAuth to fetch protected resources -========================================= +####Using OAuth to fetch protected resources + +***mYQL*** comes with ***[yahoo_oauth](https://pypi.python.org/pypi/yahoo_oauth)***, which is an OAuth library for Yahoo! APIs ```python ->>> from myql.contrib.auth import YOAuth ->>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. +>>> from yahoo_oauth import OAuth1 +>>> oauth = OAuth1(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. >>> from myql import MYQL >>> yql = MYQL(format='xml', oauth=oauth) >>> response = yql.getGUID('josue_brunel') # Deal with the response diff --git a/myql/myql.py b/myql/myql.py index e18e6b0..4185f7e 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -2,7 +2,6 @@ import logging import requests -from myql.contrib.auth import YOAuth import myql.errors From 7381067d00c41a9c8694b635e1501579c8d282b8 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 7 Jun 2015 12:18:31 +0200 Subject: [PATCH 424/582] #1: fixing badges --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 51211a1..1b341a9 100755 --- a/README.md +++ b/README.md @@ -2,12 +2,13 @@ ========= [![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) -[![Latest Version](https://pypip.in/version/myql/badge.svg)](https://pypi.python.org/pypi/myql/) -[![Downloads](https://pypip.in/download/myql/badge.svg)](https://pypi.python.org/pypi/myql) -[![Py_Versions](https://pypip.in/py_versions/myql/badge.svg)](https://pypi.python.org/pypi/myql) -[![Implementations](https://pypip.in/implementation/myql/badge.svg)](https://pypi.python.org/pypi/myql) + [![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Issues](https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg)](https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql) +[![PyPI](https://img.shields.io/pypi/v/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) +[![PyPI](https://img.shields.io/pypi/dm/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) +[![PyPI](https://img.shields.io/pypi/l/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) + mYQL is a Python wrapper of the Yahoo Query Language. **[Full Documentation](http://myql.readthedocs.org/en/latest/)** @@ -223,7 +224,7 @@ Delete records ####Using OAuth to fetch protected resources -***mYQL*** comes with ***[yahoo_oauth](https://pypi.python.org/pypi/yahoo_oauth)***, which is an OAuth library for Yahoo! APIs +***mYQL*** comes with ***[yahoo_oauth](https://pypi.python.org/pypi/myql)***, which is an OAuth library for Yahoo! APIs ```python >>> from yahoo_oauth import OAuth1 From afde5dbd3a26bbd7b76654ee80c0996f3704aa05 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 7 Jun 2015 12:19:38 +0200 Subject: [PATCH 425/582] #1 updating packages info --- dev_requirements.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index f1a29c2..7937541 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -15,4 +15,4 @@ requests-oauthlib==0.5.0 six==1.9.0 tornado==4.2 wsgiref==0.1.2 -yahoo-oauth==0.1.3 +yahoo-oauth==0.1.4 diff --git a/requirements.txt b/requirements.txt index 8331870..2e7fe64 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ rauth>=0.7.1 requests-oauthlib>=0.4.2 requests>=2.7.0 six>=1.9.0 -yahoo-oauth>=0.1.3 +yahoo-oauth>=0.1.4 From 90ecf7ec5783e1558ddb18c1959c2806a867bc0a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 7 Jun 2015 15:08:47 +0200 Subject: [PATCH 426/582] fix another IN issue --- myql/myql.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 4185f7e..f6e60e0 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -104,11 +104,13 @@ def clauseFormatter(self, cond): '''Formats conditions args is a list of ['column', 'operator', 'value'] ''' + if cond[1].lower() == 'in': if len(cond[2]) > 1: - cond[2] = "('{0}')".format(','.join(map(str,[ "{0}".format(e) for e in cond[2] ]))) + cond[2] = "({0})".format(','.join(map(str,[ "'{0}'".format(e) for e in cond[2] ]))) else: cond[2] = "({0})".format(','.join(map(str,[ "{0}".format(e) for e in cond[2] ]))) + cond = " ".join(cond) else: cond[2] = "'{0}'".format(cond[2]) From 591806781251d09cc7224d3ba7832e40ea382441 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 7 Jun 2015 15:20:29 +0200 Subject: [PATCH 427/582] dump method add --- myql/utils.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/myql/utils.py b/myql/utils.py index 77a1639..a5b82c7 100644 --- a/myql/utils.py +++ b/myql/utils.py @@ -3,28 +3,44 @@ from xml.etree import cElementTree as tree def pretty_json(data): + """Return a pretty formatted json + """ data = json.loads(data.decode('utf-8')) return json.dumps(data, indent=4, sort_keys=True) def pretty_xml(data): + """Return a pretty formated xml + """ parsed_string = minidom.parseString(data.decode('utf-8')) return parsed_string.toprettyxml(indent='\t', encoding='utf-8') def prettyfy(response, format='json'): - + """A wrapper for pretty_json and pretty_xml + """ if format=='json': return pretty_json(response.content) else : return pretty_xml(response.content) def json_write_data(json_data, filename): + """Write data in json file + """ with open(filename, 'w') as fp: json.dump(json_data, fp, indent=4, sort_keys=True, ensure_ascii=False) return True return False def json_get_data(filename): + """Get data from json file + """ with open(filename, 'r') as fp: json_data = json.load(fp) return json_data - + +def dump(response): + """Print a pretty formatted response content + """ + try: + print(pretty_json(response.content)) + except: + prrint(pretty_xml(response.content)) From cc18c0513623ca8f311418c4c6ad85874625b8c3 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 7 Jun 2015 15:20:52 +0200 Subject: [PATCH 428/582] #1: updating documentation --- README.md | 279 +++++++++++++++++++++++++++++++++++++--------- docs/index.md | 300 +++++++++++++++++++++++++++++++++++++++++--------- docs/oauth.md | 62 ++--------- docs/table.md | 2 +- mkdocs.yml | 14 +-- 5 files changed, 497 insertions(+), 160 deletions(-) diff --git a/README.md b/README.md index 1b341a9..b9bac95 100755 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ ========= [![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) - -[![Join the chat at https://gitter.im/josuebrunel/myql](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/josuebrunel/myql?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Issues](https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg)](https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql) - + [![Code Issues](https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg)](https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql) [![PyPI](https://img.shields.io/pypi/v/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/dm/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/l/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) @@ -53,32 +51,14 @@ $ pip install myql Quick Start =========== -```python ->>> import myql ->>> yql = myql.MYQL() ->>> yql.diagnostics = True # To turn diagnostics on -``` +It's important to know that **response** is a just **requests.models.Response** object. +Yes indeed, ***mYQL*** uses ***requests*** :smile: -####Disable access to community tables - -```python ->>> rep = yql.rawQuery('desc yahoo.finance.quotes ') ->>> rep.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} ->>> ->>> yql.community= True # Setting up access to community ->>> yql = myql.MYQL() ->>> rep = yql.rawQuery('desc yahoo.finance.quotes ') ->>> rep.json() -{u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} - -``` -or +By default, you have access to the **community tables**. If for whatsoever reason you would like to not have access to those tables ```python >>> import myql >>> yql = myql.MYQL(community=False) ->>> # do your magic ``` ####Changing response format (xml or json) @@ -87,13 +67,120 @@ The response format is by default ***json***. ```python >>> import myql +>>> from myql.utils import pretty_json, pretty_xml >>> yql = myql.MYQL(format='xml', community=True) ->>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') ->>> rep.text -u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' ->>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') ->>> rep.json() -{u'query': {u'count': 11, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'55998384', u'name': u'Cuvette-Ouest Department'}, {u'woeid': u'2344968', u'name': u'Cuvette Department'}, {u'woeid': u'2344973', u'name': u'Plateaux District'}, {u'woeid': u'2344974', u'name': u'Sangha'}, {u'woeid': u'2344970', u'name': u'Lekoumou'}, {u'woeid': u'2344975', u'name': u'Pool Department'}, {u'woeid': u'2344971', u'name': u'Likouala Department'}, {u'woeid': u'2344972', u'name': u'Niari Department'}, {u'woeid': u'2344976', u'name': u'Brazzaville'}, {u'woeid': u'2344967', u'name': u'Bouenza Department'}, {u'woeid': u'2344969', u'name': u'Kouilou'}]}, u'created': u'2014-08-27T04:52:38Z'}} +>>> resp = yql.rawQuery('select name, woeid from geo.states where place="Congo"') +>>> print(pretty_xml(resp.content)) + + + + + Cuvette-Ouest Department + 55998384 + + + Cuvette Department + 2344968 + + + Plateaux District + 2344973 + + + Sangha + 2344974 + + + Lekoumou + 2344970 + + + Pool Department + 2344975 + + + Likouala Department + 2344971 + + + Niari Department + 2344972 + + + Brazzaville + 2344976 + + + Bouenza Department + 2344967 + + + Kouilou + 2344969 + + + + + + +>>> resp = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') +>>> print(pretty_json(resp.content)) +{ + "query": { + "count": 11, + "created": "2015-06-07T11:58:20Z", + "lang": "en-US", + "results": { + "place": [ + { + "name": "Cuvette-Ouest Department", + "woeid": "55998384" + }, + { + "name": "Cuvette Department", + "woeid": "2344968" + }, + { + "name": "Plateaux District", + "woeid": "2344973" + }, + { + "name": "Sangha", + "woeid": "2344974" + }, + { + "name": "Lekoumou", + "woeid": "2344970" + }, + { + "name": "Pool Department", + "woeid": "2344975" + }, + { + "name": "Likouala Department", + "woeid": "2344971" + }, + { + "name": "Niari Department", + "woeid": "2344972" + }, + { + "name": "Brazzaville", + "woeid": "2344976" + }, + { + "name": "Bouenza Department", + "woeid": "2344967" + }, + { + "name": "Kouilou", + "woeid": "2344969" + } + ] + } + } +} + >>> ``` @@ -113,8 +200,58 @@ Returns table description ```python >>> response = yql.desc('weather.forecast') ->>> response.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'request': {u'select': [{u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'location'}, {u'type': u'xs:string', u'name': u'u'}]}, {u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'woeid'}, {u'type': u'xs:string', u'name': u'u'}]}]}, u'security': u'ANY', u'meta': {u'documentationURL': u'http://developer.yahoo.com/weather/', u'sampleQuery': u'select * from weather.forecast where woeid=2502265', u'description': u'Weather forecast table', u'author': u'Yahoo! Inc'}, u'hash': u'aae78b1462a6a8fbc748aec4cf292767', u'name': u'weather.forecast'}}, u'created': u'2014-08-16T19:31:51Z'}} +>>> print(pretty_json(response.content)) +{ + "query": { + "count": 1, + "created": "2015-06-07T12:00:27Z", + "lang": "en-US", + "results": { + "table": { + "hash": "aae78b1462a6a8fbc748aec4cf292767", + "meta": { + "author": "Yahoo! Inc", + "description": "Weather forecast table", + "documentationURL": "http://developer.yahoo.com/weather/", + "sampleQuery": "select * from weather.forecast where woeid=2502265" + }, + "name": "weather.forecast", + "request": { + "select": [ + { + "key": [ + { + "name": "location", + "required": "true", + "type": "xs:string" + }, + { + "name": "u", + "type": "xs:string" + } + ] + }, + { + "key": [ + { + "name": "woeid", + "required": "true", + "type": "xs:string" + }, + { + "name": "u", + "type": "xs:string" + } + ] + } + ] + }, + "security": "ANY" + } + } + } +} + >>> ``` @@ -127,25 +264,52 @@ Allows you to directly type your query >>> # deal with the response ``` -####select(table, fields, limit).where(filters, ...) -Select a table i.e *weather.forecast*. -If *table* not provided, it will use the default table. If there's no such thing as a default table, it will raise a *NoTableSelectedError* +####select(table, fields, limit).where(filters, ...) ***NB*** : A simple select doesn't return any data. Use ***GET*** instead. ```python ->>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) ->>> response.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424775', u'name': u'Canada'}}, u'created': u'2014-08-16T19:04:08Z'}} +>>> response = yql.select('geo.countries', ['name', 'code', 'woeid']).where(['name', '=', 'Canada']) +>>> print(pretty_json(response.content)) +{ + "query": { + "count": 1, + "created": "2015-06-07T12:10:39Z", + "lang": "en-US", + "results": { + "place": { + "name": "Canada", + "woeid": "23424775" + } + } + } +} + >>> ... ->>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', '=', 'Africa']) ->>> rep.json() -{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T10:52:49Z'}} ->>> ->>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) ->>> rep.json() -{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T11:22:49Z'}} +>>> response = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) +>>> from myql.utils import dump +>>> dump(response) +{ + "query": { + "count": 2, + "created": "2015-06-07T12:27:04Z", + "lang": "en-US", + "results": { + "place": [ + { + "name": "Algeria", + "woeid": "23424740" + }, + { + "name": "Angola", + "woeid": "23424745" + } + ] + } + } +} + >>> ``` @@ -155,9 +319,23 @@ Same as ***SELECT***, but instead returns data. **REMINDER** : Some tables require a **where clause**, therefore ***GET*** won't work on those tables, use *select(...).where(...)* instead . ```python ->>> yql.get('geo.countries', ['name', 'woeid'], 1) ->>> rep.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} +>>> from myql.utils import dump +>>> response = yql.get('geo.countries', ['name', 'woeid'], 1) +>>> dump(response) +{ + "query": { + "count": 1, + "created": "2015-06-07T12:29:01Z", + "lang": "en-US", + "results": { + "place": { + "name": "Sao Tome and Principe", + "woeid": "23424966" + } + } + } +} + >>> ``` @@ -222,9 +400,9 @@ Delete records ``` -####Using OAuth to fetch protected resources +####Using OAuth -***mYQL*** comes with ***[yahoo_oauth](https://pypi.python.org/pypi/myql)***, which is an OAuth library for Yahoo! APIs +***mYQL*** comes with ***[yahoo_oauth](https://pypi.python.org/pypi/myql)***, which is an OAuth library for Yahoo! APIs. ```python >>> from yahoo_oauth import OAuth1 @@ -243,6 +421,7 @@ Delete records * Fixed issue related to date in StockRetriver.get_historical_info [#107](https://github.com/josuebrunel/myql/issues/107) * Fixed issue with **IN** condition in **where** clause [#106](https://github.com/josuebrunel/myql/issues/107) * Fix definition of raw_input for python3 [#105](https://github.com/josuebrunel/myql/issues/105) +* Yahoo-OAuth included as main oauth library [#112](https://github.com/josuebrunel/myql/issues/112) ##### 1.2.2 ------- diff --git a/docs/index.md b/docs/index.md index f4f8bf9..c0c9c95 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,8 +3,7 @@ mYQL mYQL is a Python wrapper of the Yahoo Query Language. -Yahoo! Query Language Documentation and Support -=============================================== +## Yahoo! Query Language Documentation and Support * [Yahoo! Query Language](http://developer.yahoo.com/yql/) * [Yahoo! Developer Network](http://developer.yahoo.com) @@ -22,47 +21,136 @@ $ pip install myql Quick Start =========== -```python ->>> import myql ->>> yql = myql.MYQL() ->>> yql.diagnostics = True # To turn diagnostics on -``` - -####Disable access to community tables +It's important to know that **response** is a just **requests.models.Response** object. +Yes indeed, ***mYQL*** uses ***requests*** :smile: -```python ->>> rep = yql.rawQuery('desc yahoo.finance.quotes ') ->>> rep.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'src': u'http://www.datatables.org/yahoo/finance/yahoo.finance.quotes.xml', u'hash': u'061616a1c033ae89aaf2cbe83790b979', u'name': u'yahoo.finance.quotes', u'request': {u'select': {u'key': {u'required': u'true', u'type': u'xs:string', u'name': u'symbol'}}}, u'meta': {u'sampleQuery': u'\n\t\t\tselect * from yahoo.finance.quotes where symbol in ("YHOO","AAPL","GOOG","MSFT")\n\t\t'}, u'security': u'ANY'}}, u'created': u'2014-08-24T11:26:48Z'}} ->>> ->>> yql.community= True # Setting up access to community ->>> yql = myql.MYQL() ->>> rep = yql.rawQuery('desc yahoo.finance.quotes ') ->>> rep.json() -{u'error': {u'lang': u'en-US', u'description': u'No definition found for Table yahoo.finance.quotes'}} - -``` -or +By default, you have access to the **community tables**. If for whatsoever reason you would like to not have access to those tables ```python >>> import myql >>> yql = myql.MYQL(community=False) ->>> # do your magic ``` -####Changing response format (xml or json) +#### Response format (xml or json) The response format is by default ***json***. ```python >>> import myql +>>> from myql.utils import pretty_json, pretty_xml >>> yql = myql.MYQL(format='xml', community=True) ->>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"') ->>> rep.text -u'\nCuvette-Ouest Department55998384Cuvette Department2344968Plateaux District2344973Sangha2344974Lekoumou2344970Pool Department2344975Likouala Department2344971Niari Department2344972Brazzaville2344976Bouenza Department2344967Kouilou2344969\n\n' ->>> rep = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') ->>> rep.json() -{u'query': {u'count': 11, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'55998384', u'name': u'Cuvette-Ouest Department'}, {u'woeid': u'2344968', u'name': u'Cuvette Department'}, {u'woeid': u'2344973', u'name': u'Plateaux District'}, {u'woeid': u'2344974', u'name': u'Sangha'}, {u'woeid': u'2344970', u'name': u'Lekoumou'}, {u'woeid': u'2344975', u'name': u'Pool Department'}, {u'woeid': u'2344971', u'name': u'Likouala Department'}, {u'woeid': u'2344972', u'name': u'Niari Department'}, {u'woeid': u'2344976', u'name': u'Brazzaville'}, {u'woeid': u'2344967', u'name': u'Bouenza Department'}, {u'woeid': u'2344969', u'name': u'Kouilou'}]}, u'created': u'2014-08-27T04:52:38Z'}} +>>> resp = yql.rawQuery('select name, woeid from geo.states where place="Congo"') +>>> print(pretty_xml(resp.content)) + + + + + Cuvette-Ouest Department + 55998384 + + + Cuvette Department + 2344968 + + + Plateaux District + 2344973 + + + Sangha + 2344974 + + + Lekoumou + 2344970 + + + Pool Department + 2344975 + + + Likouala Department + 2344971 + + + Niari Department + 2344972 + + + Brazzaville + 2344976 + + + Bouenza Department + 2344967 + + + Kouilou + 2344969 + + + + + + +>>> resp = yql.rawQuery('select name, woeid from geo.states where place="Congo"', format='json') +>>> print(pretty_json(resp.content)) +{ + "query": { + "count": 11, + "created": "2015-06-07T11:58:20Z", + "lang": "en-US", + "results": { + "place": [ + { + "name": "Cuvette-Ouest Department", + "woeid": "55998384" + }, + { + "name": "Cuvette Department", + "woeid": "2344968" + }, + { + "name": "Plateaux District", + "woeid": "2344973" + }, + { + "name": "Sangha", + "woeid": "2344974" + }, + { + "name": "Lekoumou", + "woeid": "2344970" + }, + { + "name": "Pool Department", + "woeid": "2344975" + }, + { + "name": "Likouala Department", + "woeid": "2344971" + }, + { + "name": "Niari Department", + "woeid": "2344972" + }, + { + "name": "Brazzaville", + "woeid": "2344976" + }, + { + "name": "Bouenza Department", + "woeid": "2344967" + }, + { + "name": "Kouilou", + "woeid": "2344969" + } + ] + } + } +} + >>> ``` @@ -82,8 +170,58 @@ Returns table description ```python >>> response = yql.desc('weather.forecast') ->>> response.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'table': {u'request': {u'select': [{u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'location'}, {u'type': u'xs:string', u'name': u'u'}]}, {u'key': [{u'required': u'true', u'type': u'xs:string', u'name': u'woeid'}, {u'type': u'xs:string', u'name': u'u'}]}]}, u'security': u'ANY', u'meta': {u'documentationURL': u'http://developer.yahoo.com/weather/', u'sampleQuery': u'select * from weather.forecast where woeid=2502265', u'description': u'Weather forecast table', u'author': u'Yahoo! Inc'}, u'hash': u'aae78b1462a6a8fbc748aec4cf292767', u'name': u'weather.forecast'}}, u'created': u'2014-08-16T19:31:51Z'}} +>>> print(pretty_json(response.content)) +{ + "query": { + "count": 1, + "created": "2015-06-07T12:00:27Z", + "lang": "en-US", + "results": { + "table": { + "hash": "aae78b1462a6a8fbc748aec4cf292767", + "meta": { + "author": "Yahoo! Inc", + "description": "Weather forecast table", + "documentationURL": "http://developer.yahoo.com/weather/", + "sampleQuery": "select * from weather.forecast where woeid=2502265" + }, + "name": "weather.forecast", + "request": { + "select": [ + { + "key": [ + { + "name": "location", + "required": "true", + "type": "xs:string" + }, + { + "name": "u", + "type": "xs:string" + } + ] + }, + { + "key": [ + { + "name": "woeid", + "required": "true", + "type": "xs:string" + }, + { + "name": "u", + "type": "xs:string" + } + ] + } + ] + }, + "security": "ANY" + } + } + } +} + >>> ``` @@ -96,25 +234,52 @@ Allows you to directly type your query >>> # deal with the response ``` -####select(table, fields, limit).where(filters, ...) -Select a table i.e *weather.forecast*. -If *table* not provided, it will use the default table. If there's no such thing as a default table, it will raise a *NoTableSelectedError* +####select(table, fields, limit).where(filters, ...) ***NB*** : A simple select doesn't return any data. Use ***GET*** instead. ```python ->>> response = yql.select('geo.countries', [name, code, woeid]).where(['name', '=', 'Canada']) ->>> response.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424775', u'name': u'Canada'}}, u'created': u'2014-08-16T19:04:08Z'}} +>>> response = yql.select('geo.countries', ['name', 'code', 'woeid']).where(['name', '=', 'Canada']) +>>> print(pretty_json(response.content)) +{ + "query": { + "count": 1, + "created": "2015-06-07T12:10:39Z", + "lang": "en-US", + "results": { + "place": { + "name": "Canada", + "woeid": "23424775" + } + } + } +} + >>> ... ->>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', '=', 'Africa']) ->>> rep.json() -{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T10:52:49Z'}} ->>> ->>> rep = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) ->>> rep.json() -{u'query': {u'count': 2, u'lang': u'en-US', u'results': {u'place': [{u'woeid': u'23424740', u'name': u'Algeria'}, {u'woeid': u'23424745', u'name': u'Angola'}]}, u'created': u'2014-08-17T11:22:49Z'}} +>>> response = yql.select('geo.countries', ['name', 'woeid'], 2).where(['place', 'in', ('Africa', 'Europe')]) +>>> from myql.utils import dump +>>> dump(response) +{ + "query": { + "count": 2, + "created": "2015-06-07T12:27:04Z", + "lang": "en-US", + "results": { + "place": [ + { + "name": "Algeria", + "woeid": "23424740" + }, + { + "name": "Angola", + "woeid": "23424745" + } + ] + } + } +} + >>> ``` @@ -124,9 +289,23 @@ Same as ***SELECT***, but instead returns data. **REMINDER** : Some tables require a **where clause**, therefore ***GET*** won't work on those tables, use *select(...).where(...)* instead . ```python ->>> yql.get('geo.countries', ['name', 'woeid'], 1) ->>> rep.json() -{u'query': {u'count': 1, u'lang': u'en-US', u'results': {u'place': {u'woeid': u'23424966', u'name': u'Sao Tome and Principe'}}, u'created': u'2014-08-17T10:32:25Z'}} +>>> from myql.utils import dump +>>> response = yql.get('geo.countries', ['name', 'woeid'], 1) +>>> dump(response) +{ + "query": { + "count": 1, + "created": "2015-06-07T12:29:01Z", + "lang": "en-US", + "results": { + "place": { + "name": "Sao Tome and Principe", + "woeid": "23424966" + } + } + } +} + >>> ``` @@ -191,13 +370,30 @@ Delete records ``` -Using OAuth to fetch protected resources -========================================= +## Using OAuth + +***mYQL*** comes with ***[yahoo_oauth](https://pypi.python.org/pypi/myql)***, which is an OAuth library for Yahoo! APIs. ```python ->>> from myql.contrib.auth import YOAuth ->>> oauth = YOAuth(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. +>>> from yahoo_oauth import OAuth1 +>>> oauth = OAuth1(None, None, from_file='credentials.json') # only consumer_key and consumer_secret are required. >>> from myql import MYQL >>> yql = MYQL(format='xml', oauth=oauth) >>> response = yql.getGUID('josue_brunel') # Deal with the response ``` + +## Utils + +***mYQL*** comes with some useful functions, such as: + +#### *prettyty(response, format)* +    According to the format, call *pretty_json* or *pretty_xml* + +#### *pretty_json(response.content)* +    prettyfy a JSON response content + +#### *pretty_xml(response.content)* +    Prettyfy a XML response content + +#### *dump(response)* +    Print a prettyfied response diff --git a/docs/oauth.md b/docs/oauth.md index e5f1afe..ad2a5c9 100644 --- a/docs/oauth.md +++ b/docs/oauth.md @@ -1,57 +1,19 @@ -YOAuth as Yahoo OAuth +Yahoo-OAuth ===================== -Before going any further i would like to thank [Darren Kempiners](https://github.com/dkempiners) and [Andrew Martin](https://github.com/almartin82) for their help. +The ***YOAuth*** module is not supported anymore. mYQL comes with + ***[Yahoo-OAuth](http://yahoo-oauth.readthedocs.org/en/master/)***. + From now on, this will be the library to use to when you want to use **OAuth**. + Nothing has really changed anyway, since *yahoo-oauth* is based on *YOAuth*. -The ***YOAuth*** class is used to generates an oauth *session* which will be used in your requests to the *YQL Service* - -### **Definition** - -#### *YOAuth(consumer_key, consumer_secret, \*\*kwargs)* - -* ***consumer_key*** : Client Key of your application. -* ***consumer_secret*** : Client Secret of your application -* ***acess_token*** : The Access Token -* ***acess_token_secret*** : The Acess Token Secret -* ***session_handler*** : The OAuth Session Handler which is required when refreshing the token -* ***from_file*** : File containing the credentials. - -The **minimum** information required in a ***credentials file*** are the ***consumer_key*** and the ***consumer_secret***. -If ***from_file*** is provided, the class will be instanciated with data within this file. - -```python ->>> from myql.contrib.auth import YOAuth ->>> oauth = YOAuth('khdhkfhbb7rit93ffhbfh', 'urysfjue76885hgf') -``` - -Using a credentials json file - -*credentials.json* -```json -{ - "consumer_key" : "khdhkfhbb7rit93ffhbfh", - "consumer_secret": "urysfjue76885hgf" -} -``` +You can read the full documentation [here](http://yahoo-oauth.readthedocs.org/en/master/) ```python ->>> oauth = YOAuth(None, None, from_file=os.path.realpath('credentials.json')) +>>> import myql +>>> from yahoo_oauth import OAuth1 +>>> oauth = OAuth1(None, None, from_file='credentials.json') +>>> yql = myql.MYQL(format='xml',oauth=oauth) +>>> response = yql.getGUID('josue_brunel') +... ``` -### **Methods** - -- #### *OAuth.json_get_data(filename)* - -Return a dict containing the credentials - -- #### *OAuth.json_write_data(json_data, filename)* - -Update the credentials file - -- #### *OAuth.token_is_valid()* - -Check if the token is still valid - -- #### *OAuth.refresh_token()* - -Refresh the expired token \ No newline at end of file diff --git a/docs/table.md b/docs/table.md index b62443d..7f0fa9c 100644 --- a/docs/table.md +++ b/docs/table.md @@ -162,7 +162,7 @@ This class represents a stored function. Read the full documentation [here](http As ***Binder***, ***BinderFunction*** is a subclass of ***BaseBinder***. They both share the same methods -## Using MetaClasses to define a Table +## The MetaClass API ***BinderModel*** and ***TableModel*** are the only classes to keep in mind here. They're respectively subclasses of ***BinderMeta*** and ***TableMeta***. Those last two help providing a powerful API to define YQL Table. diff --git a/mkdocs.yml b/mkdocs.yml index 084b09d..e84cdc5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,12 +3,12 @@ repo_url: https://github.com/josuebrunel/myql site_url: http://myql.readthedocs.org/en/latest/myql/ site_description: MYQL documentation site_author: Josue Kouka -google_analytics: ['UA-32441224-4', 'readthedocs.org'] +google_analytics: ['UA-32441224-4', 'myql.readthedocs.org'] pages: -- [index.md, Home] -- [myql.md, MYQL] -- [stockscraper.md, StockScraper] -- [oauth.md, YOAuth] -- [table.md, Open Table] -- [contrib.md, Contribute] +- Home : index.md +- mYQL : myql.md +- StockScraper : stockscraper.md +- Yahoo-OAuth : oauth.md +- Open Tables : table.md +- Contribute : contrib.md theme: readthedocs From 09fa2bd0bcf047e105153c3ca71963a9a0382bdf Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 7 Jun 2015 15:23:29 +0200 Subject: [PATCH 429/582] cleaning the contrib house from auth package --- myql/contrib/__init__.py | 1 - myql/contrib/auth/__init__.py | 1 - myql/contrib/auth/yoauth.py | 120 ---------------------------------- 3 files changed, 122 deletions(-) delete mode 100755 myql/contrib/auth/__init__.py delete mode 100644 myql/contrib/auth/yoauth.py diff --git a/myql/contrib/__init__.py b/myql/contrib/__init__.py index a38209b..532bae6 100755 --- a/myql/contrib/__init__.py +++ b/myql/contrib/__init__.py @@ -1,4 +1,3 @@ -from myql.contrib import auth from myql.contrib import table from myql.contrib import stockscraper diff --git a/myql/contrib/auth/__init__.py b/myql/contrib/auth/__init__.py deleted file mode 100755 index 67e22e7..0000000 --- a/myql/contrib/auth/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from myql.contrib.auth.yoauth import YOAuth diff --git a/myql/contrib/auth/yoauth.py b/myql/contrib/auth/yoauth.py deleted file mode 100644 index 990ccbb..0000000 --- a/myql/contrib/auth/yoauth.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -YOAuth is inspired from Darren Kempiners YahooAPI https://github.com/dkempiners/python-yahooapi/blob/master/yahooapi.py -""" - -from __future__ import absolute_import - -import json -import time -import logging -import webbrowser - -from rauth import OAuth1Service -from rauth.utils import parse_utf8_qsl - -from myql.utils import json_write_data, json_get_data - -try: - input = raw_input -except (NameError,): - pass - - -BASE_URL = "http://query.yahooapis.com/v1/yql" -REQUEST_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_request_token" -ACCESS_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/get_token" -AUTHORIZE_TOKEN_URL = "https://api.login.yahoo.com/oauth/v2/request_auth?oauth_token=" -CALLBACK_URI = 'oob' - -logging.basicConfig(level=logging.DEBUG, format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s") -logging.getLogger(__name__) - - -class YOAuth(object): - """ - """ - def __init__(self, consumer_key, consumer_secret, **kwargs): - """ - consumer_key : client key - consumer_secret : client secret - access_token : access token - access_token_secret : access token secret - from_file : file containing the credentials - """ - if kwargs.get('from_file'): - logging.debug("Checking ") - self.from_file = kwargs.get('from_file') - json_data = json_get_data(self.from_file) - vars(self).update(json_data) - else: - self.consumer_key = consumer_key - self.consumer_secret = consumer_secret - vars(self).update(kwargs) - - # Init OAuth - self.oauth = OAuth1Service( - consumer_key = self.consumer_key, - consumer_secret = self.consumer_secret, - name = "yahoo", - request_token_url = REQUEST_TOKEN_URL, - access_token_url = ACCESS_TOKEN_URL, - authorize_url = AUTHORIZE_TOKEN_URL, - base_url = BASE_URL - ) - - if vars(self).get('access_token') and vars(self).get('access_token_secret') and vars(self).get('session_handle'): - if not self.token_is_valid(): - self.session = self.refresh_token() - else: - # Fetching request token/token_secret - request_token, request_token_secret = self.oauth.get_request_token(params={'oauth_callback': CALLBACK_URI}) - logging.debug("REQUEST_TOKEN = {0}\n REQUEST_TOKEN_SECRET = {1}\n".format(request_token, request_token_secret)) - #authorize_url = self.oauth.get_authorize_url(request_token) - authorize_url = AUTHORIZE_TOKEN_URL+request_token - logging.debug(authorize_url) - webbrowser.open(authorize_url) - verifier = input("Enter verifier : ") - logging.debug("VERIFIER = {0}".format(verifier)) - - self.token_time = time.time() - raw_acess = self.oauth.get_raw_access_token(request_token, request_token_secret, params={"oauth_verifier": verifier}) - parsed_acess = parse_utf8_qsl(raw_acess.content) - - self.access_token = parsed_acess['oauth_token'] - self.access_token_secret = parsed_acess['oauth_token_secret'] - self.session_handle = parsed_acess['oauth_session_handle'] - - self.session = self.oauth.get_session((self.access_token, self.access_token_secret)) - - json_data.update({ - 'access_token' : self.access_token, - 'access_token_secret' : self.access_token_secret, - 'session_handle' : self.session_handle, - 'token_time' : self.token_time - }) - - json_write_data(json_data, self.from_file) - - def refresh_token(self,): - """Refresh access token - """ - logging.debug("REFRESHING TOKEN") - self.token_time = time.time() - self.access_token, self.access_token_secret = self.oauth.get_access_token(self.access_token, self.access_token_secret, params={"oauth_session_handle": self.session_handle}) - - session = self.oauth.get_session((self.access_token, self.access_token_secret)) - - return session - - def token_is_valid(self,): - """Check the validity of the token :3600s - """ - elapsed_time = time.time() - self.token_time - logging.debug("ELAPSED TIME : {0}".format(elapsed_time)) - if elapsed_time > 3540: # 1 minute before it expires - logging.debug("TOKEN HAS EXPIRED") - return False - - logging.debug("TOKEN IS STILL VALID") - return True - From 834746ee10ddb99e3dfb5bbb9c0ee7bf7850fd2d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 7 Jun 2015 15:30:27 +0200 Subject: [PATCH 430/582] fix yahoo_oauth link issue --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9bac95..803774c 100755 --- a/README.md +++ b/README.md @@ -402,7 +402,7 @@ Delete records ####Using OAuth -***mYQL*** comes with ***[yahoo_oauth](https://pypi.python.org/pypi/myql)***, which is an OAuth library for Yahoo! APIs. +***mYQL*** comes with ***[yahoo_oauth](http://yahoo-oauth.readthedocs.org/en/master/)***, which is an OAuth library for Yahoo! APIs. ```python >>> from yahoo_oauth import OAuth1 From a887a0257f6ce0e9ae612ca0c21a7d3ae6e6f36b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 7 Jun 2015 15:40:37 +0200 Subject: [PATCH 431/582] fix redanduncy in readme --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index 803774c..e91987e 100755 --- a/README.md +++ b/README.md @@ -27,20 +27,6 @@ Yahoo! Query Language Documentation and Support * YQL Open Table (Classes and Metaclasses) Generator * Response prettyfier -mYQL -========= - -mYQL is a Python wrapper of the Yahoo Query Language. - -Yahoo! Query Language Documentation and Support -=============================================== - -* [Yahoo! Query Language](http://developer.yahoo.com/yql/) -* [Yahoo! Developer Network](http://developer.yahoo.com) -* [Yahoo! Application Platform](http://developer.yahoo.com/yap/) -* [Yahoo! Social APIs](http://developer.yahoo.com/social/) -* [Yahoo! Query Language Console](https://developer.yahoo.com/yql/console/) - Installation ============ From cc15b3ff1b49940d710313974c4752c2f57a5033 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 8 Jun 2015 12:39:30 +0200 Subject: [PATCH 432/582] #1: updating documentation --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e91987e..6bf2df2 100755 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ Methods ------- ####use(yql_table_url,name=yql_table_name) -Changes the data provider +Maps a table name to the URL of an Open Data Table. ```python >>> yql.use('http://www.josuebrunel.org//users.xml', name='myusers') diff --git a/docs/index.md b/docs/index.md index c0c9c95..1a7af9c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -159,7 +159,7 @@ Methods ------- ####use(yql_table_url,name=yql_table_name) -Changes the data provider +Maps a table name to the URL of an Open Data Table. ```python >>> yql.use('http://www.josuebrunel.org//users.xml', name='myusers') From 96f068bf8f5c949b5bcc60ffe9927fed9353699e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 8 Jun 2015 21:23:17 +0200 Subject: [PATCH 433/582] #117: setting up coverage --- .coveragerc | 28 ++++++++++++++++++++++++++++ .travis.yml | 21 +++++++++++++-------- README.md | 2 +- setup.py | 1 + 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..33b21e0 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,28 @@ +# .coveragerc to control coverage.py +[run] +branch = True +source=myql + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +ignore_errors = True + +[html] +directory = coverage_html_report + diff --git a/.travis.yml b/.travis.yml index 8d090c3..5ff00c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,21 @@ language: python python: -- '2.7' -- '3.2' -- '3.3' -- '3.4' -- pypy -- pypy3 -install: pip install -r requirements.txt -script: "./run_tests.sh" + - '2.7' + - '3.2' + - '3.3' + - '3.4' + - pypy + - pypy3 +install: + - pip install -r requirements.txt + - pip install coverage coveralls +script: + - coverage run --source=myql -m unittest discover env: global: secure: hXpkZqclyUXMX586jS4BtJmYsrszVr/jWbMiLWLOZ2z7n5LK9INziKkOqYu0JTBskAHVEmhxJ7oNOD9gI/06btLO5NLBONN3qtsHq4UruCo0Zlx2BgaDfR5FqnrVqk+fMbLCbzWgo0wShM4o8jTGy7l22xqhhYAsuubmayqEfPk= before_install: - openssl aes-256-cbc -K $encrypted_60f42691f4f2_key -iv $encrypted_60f42691f4f2_iv -in credentials.json.enc -out credentials.json -d +after_install: + - coveralls diff --git a/README.md b/README.md index 6bf2df2..f15a20a 100755 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![PyPI](https://img.shields.io/pypi/v/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/dm/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/l/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) - +[![Coverage Status](https://coveralls.io/repos/josuebrunel/myql/badge.svg)](https://coveralls.io/r/josuebrunel/myql) mYQL is a Python wrapper of the Yahoo Query Language. **[Full Documentation](http://myql.readthedocs.org/en/latest/)** diff --git a/setup.py b/setup.py index 9fed9af..5e38a8b 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ def read(fname): download_url = "https://github.com/josuebrunel/myql/archive/{0}.tar.gz".format(__version__), keywords = ['myql', 'yql', 'yahoo', 'query', 'language'], packages = find_packages(), + tests_suite="tests", classifiers = [ 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', From 9d55db551891d63610fea22a6633dcdc003b7139 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 8 Jun 2015 21:35:22 +0200 Subject: [PATCH 434/582] running coveragealls after success --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5ff00c7..0680d61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,5 +17,5 @@ env: before_install: - openssl aes-256-cbc -K $encrypted_60f42691f4f2_key -iv $encrypted_60f42691f4f2_iv -in credentials.json.enc -out credentials.json -d -after_install: +after_success: - coveralls From 524464a38c6385bdfdc97ca31a9fd0b37fe880fa Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 8 Jun 2015 21:44:07 +0200 Subject: [PATCH 435/582] testing coverage badge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f15a20a..f1fc012 100755 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ [![PyPI](https://img.shields.io/pypi/v/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/dm/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/l/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) -[![Coverage Status](https://coveralls.io/repos/josuebrunel/myql/badge.svg)](https://coveralls.io/r/josuebrunel/myql) +[![Coverage Status](https://coveralls.io/repos/josuebrunel/myql/badge.svg?branch=testing)](https://coveralls.io/r/josuebrunel/myql?branch=testing) + mYQL is a Python wrapper of the Yahoo Query Language. **[Full Documentation](http://myql.readthedocs.org/en/latest/)** From 6c87ec2f41941ee1a351faef7b1601058512f3c9 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Mon, 8 Jun 2015 22:00:54 +0200 Subject: [PATCH 436/582] #117: cleaning the coverage house --- myql/contrib/table/example.py | 36 ----------------------------------- myql/contrib/table/mytable.py | 29 ---------------------------- 2 files changed, 65 deletions(-) delete mode 100644 myql/contrib/table/example.py delete mode 100644 myql/contrib/table/mytable.py diff --git a/myql/contrib/table/example.py b/myql/contrib/table/example.py deleted file mode 100644 index 2fbaa9a..0000000 --- a/myql/contrib/table/example.py +++ /dev/null @@ -1,36 +0,0 @@ -from binder import BinderModel, InputKey, PagingPage, PagingUrl, InputValue, BinderFunction -from table import TableModel, BinderFrom - -class SelectBinder(BinderModel): - name = 'select' - itemPath = 'products.product' - produces = 'xml' - pollingFrequencySeconds = 30 - urls = ['http://lol.com/services?artist={artis}','http://lol.com/services/song={song}'] - paging = PagingPage({'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) - artist = InputKey(id='artist', type='xs:string', paramType='path') - song = InputKey(id='song', type='xs:string', paramType='path', required=True) - -class InsertBinder(BinderModel): - name = 'insert' - itemPath = 'products.product' - produces = 'xml' - pollingFrequencySeconds = 30 - urls = ['http://lol.com/services?artist={artis}','http://lol.com/services/song={song}'] - paging = PagingUrl(nextpage={'path':'yqlsearch.nextpage'}) - artist = InputKey(id='artist', type='xs:string', paramType='path') - song = InputValue(id='song', type='xs:string', paramType='path', required=True) - - -class TestTable(TableModel): - name = 'Test' - author = 'Josue Kouka' - apiKeyURL = 'http://josuebrunel.org/api' - documentationURL = 'http://josuebrunel.org/doc.html' - description = "Just a test table" - sampleQuery = ['SELECT * FROM mytable','SELECT name FROM mytable WHERE id=4656', "SELECT * FROM mytable WHERE name='Josh'"] - select = BinderFrom(SelectBinder) - insert = BinderFrom(InsertBinder) - func1 = BinderFunction('concat', func_code="console.log('Hello Josh!!!')") - -TestTable.table.save(name='Example') diff --git a/myql/contrib/table/mytable.py b/myql/contrib/table/mytable.py deleted file mode 100644 index dac38a3..0000000 --- a/myql/contrib/table/mytable.py +++ /dev/null @@ -1,29 +0,0 @@ -import pdb - -try: - from myql.contrib.table import BinderMeta, TableMeta, BinderModel, BinderKey, BinderPage, TableModel -except: - from binder import BinderModel, BinderPage, BinderKey - from table import TableModel - -class SelectBinder(BinderModel): - name = 'select' - itemPath = 'products.product' - produces = 'xml' - pollingFrequencySeconds = 30 - urls = ['http://lol.com/services?artist=$','http://lol.com/services/song=$'] - paging = BinderPage('page', {'id': 'ItemPage', 'default': '1'}, {'id':'Count' ,'max':'25'},{'default': '10'}) - artist = BinderKey(id='artist', type='xs:string', paramType='path') - song = BinderKey(id='song', type='xs:string', paramType='path', required='true') - - -class TestTable(TableModel): - name = 'Test' - author = 'Josue Kouka' - apiKeyURL = 'http://josuebrunel.org/api' - documentationURL = 'http://josuebrunel.org/doc.html' - sampleQuery = 'SELECT * FROM mytable' - select = SelectBinder - - -print(TestTable.toxml()) From ba427817085d725aec1633595a5594838756f3c0 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 9 Jun 2015 04:48:21 +0200 Subject: [PATCH 437/582] cleaning the utils house --- dev_requirements.txt | 43 +++++++++++++++++++++++++------------------ myql/utils.py | 24 ++++++------------------ 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 7937541..f087e44 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,18 +1,25 @@ -Jinja2==2.7.3 -Markdown==2.6.2 -MarkupSafe==0.23 -PyYAML==3.11 -backports.ssl-match-hostname==3.4.0.2 -certifi==2015.04.28 -click==4.0 -ghp-import==0.4.1 -livereload==2.4.0 -mkdocs==0.13.3 -oauthlib==0.7.2 -rauth==0.7.1 -requests==2.7.0 -requests-oauthlib==0.5.0 -six==1.9.0 -tornado==4.2 -wsgiref==0.1.2 -yahoo-oauth==0.1.4 +Jinja2>=2.7.3 +Markdown>=2.6.2 +MarkupSafe>=0.23 +PyYAML>=3.11 +backports.ssl-match-hostname>=3.4.0.2 +certifi>=2015.04.28 +click>=4.0 +coverage>=3.7.1 +coveralls>=0.5 +docopt>=0.6.2 +flake8>=2.4.1 +ghp-import>=0.4.1 +livereload>=2.4.0 +mccabe>=0.3 +mkdocs>=0.13.3 +oauthlib>=0.7.2 +pep8>=1.5.7 +pyflakes>=0.8.1 +rauth>=0.7.1 +requests>=2.7.0 +requests-oauthlib>=0.5.0 +six>=1.9.0 +tornado>=4.2 +wsgiref>=0.1.2 +yahoo-oauth>=0.1.4 diff --git a/myql/utils.py b/myql/utils.py index a5b82c7..6df9a45 100644 --- a/myql/utils.py +++ b/myql/utils.py @@ -1,6 +1,6 @@ import json from xml.dom import minidom -from xml.etree import cElementTree as tree + def pretty_json(data): """Return a pretty formatted json @@ -8,34 +8,22 @@ def pretty_json(data): data = json.loads(data.decode('utf-8')) return json.dumps(data, indent=4, sort_keys=True) + def pretty_xml(data): """Return a pretty formated xml """ parsed_string = minidom.parseString(data.decode('utf-8')) return parsed_string.toprettyxml(indent='\t', encoding='utf-8') + def prettyfy(response, format='json'): """A wrapper for pretty_json and pretty_xml """ - if format=='json': + if format == 'json': return pretty_json(response.content) - else : + else: return pretty_xml(response.content) -def json_write_data(json_data, filename): - """Write data in json file - """ - with open(filename, 'w') as fp: - json.dump(json_data, fp, indent=4, sort_keys=True, ensure_ascii=False) - return True - return False - -def json_get_data(filename): - """Get data from json file - """ - with open(filename, 'r') as fp: - json_data = json.load(fp) - return json_data def dump(response): """Print a pretty formatted response content @@ -43,4 +31,4 @@ def dump(response): try: print(pretty_json(response.content)) except: - prrint(pretty_xml(response.content)) + print(pretty_xml(response.content)) From d96c74612972111022f5cfc6a263b8e0a64d4bd6 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 9 Jun 2015 04:56:35 +0200 Subject: [PATCH 438/582] test_desc added --- tests/tests.py | 60 +++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 50305fc..70f18be 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,7 +1,8 @@ from __future__ import absolute_import -import os, logging, time, json -import pdb +import os +import logging +import json import unittest from xml.dom import minidom from xml.etree import cElementTree as xtree @@ -9,17 +10,14 @@ from yahoo_oauth import OAuth1 from myql import MYQL -from myql.utils import pretty_xml, pretty_json +from myql.utils import pretty_xml, pretty_json, prettyfy from myql.contrib.table import Table -from myql.contrib.table import Base, BaseInput +from myql.contrib.table import BaseInput from myql.contrib.table import Binder, BinderFunction, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset from myql.contrib.stockscraper import StockRetriever -import readline, rlcompleter -readline.parse_and_bind('tab: complete') - logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logging.getLogger('Test-mYQL') @@ -30,6 +28,7 @@ def json_write_data(json_data, filename): return True return False + def json_get_data(filename): with open(filename, 'r') as fp: json_data = json.load(fp) @@ -37,7 +36,7 @@ def json_get_data(filename): class TestMYQL(unittest.TestCase): - + def setUp(self,): self.yql = MYQL(format='json',community=True) self.insert_result = None @@ -45,6 +44,11 @@ def setUp(self,): def tearDown(self): pass + def test_desc(self,): + response = self.yql.desc('weather.forecast') + logging.debug(prettyfy(response,'json')) + self.assertEqual(response.status_code, 200) + def test_use(self): self.yql.use('http://www.josuebrunel.org/users.xml',name='users') response = self.yql.rawQuery('select * from users') @@ -53,7 +57,7 @@ def test_use(self): logging.debug(pretty_json(response.content)) except (Exception,) as e: logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_raw_query(self,): response = self.yql.rawQuery('select name, woeid from geo.states where place="Congo"') @@ -61,7 +65,7 @@ def test_raw_query(self,): logging.debug(pretty_json(response.content)) except (Exception,) as e: logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_get(self,): self.yql.format = 'xml' @@ -71,7 +75,7 @@ def test_get(self,): logging.debug(pretty_xml(response.content)) except (Exception,) as e: logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_select(self,): response = self.yql.select('geo.countries', ['name', 'code', 'woeid']).where(['name', '=', 'Canada']) @@ -79,7 +83,7 @@ def test_select(self,): logging.debug(pretty_json(response.content)) except (Exception,) as e: logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_select_in(self,): response = self.yql.select('yahoo.finance.quotes').where(['symbol','in',("YHOO","AAPL","GOOG")]) @@ -87,7 +91,7 @@ def test_select_in(self,): logging.debug(pretty_json(response.content)) except (Exception,) as e: logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_select_in_2(self,): response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','IN',('select woeid from geo.places(1) where text="Paris"',)]) @@ -95,7 +99,7 @@ def test_select_in_2(self,): logging.debug(pretty_json(response.content)) except (Exception,) as e: logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_1_insert(self,): response = self.yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) @@ -108,7 +112,7 @@ def test_1_insert(self,): logging.error(response.content) logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_2_check_insert(self,): json_data = json_get_data('yql_storage.json') @@ -119,7 +123,7 @@ def test_2_check_insert(self,): logging.error(response.content) logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_3_update(self,): json_data = json_get_data('yql_storage.json') @@ -130,7 +134,7 @@ def test_3_update(self,): logging.error(response.content) logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) def test_4_delete(self,): json_data = json_get_data('yql_storage.json') @@ -141,7 +145,7 @@ def test_4_delete(self,): logging.error(response.content) logging.error(e) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) @@ -157,8 +161,8 @@ def test_get_guid(self,): oauth = OAuth1(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) response = yql.getGUID('josue_brunel') - logging.debug(response.content) - self.assertEqual(response.status_code,200) + logging.debug(pretty_json(response.content)) + self.assertEqual(response.status_code, 200) def test_yahoo_fantasy_sport(self,): oauth = OAuth1(None, None, from_file='credentials.json') @@ -167,7 +171,7 @@ def test_yahoo_fantasy_sport(self,): year = '2015-05-05' for team in teams: response = yql.select('fantasysports.teams.roster').where(['team_key','=',team],['date','=',year]) - self.assertEqual(response.status_code,200) + self.assertEqual(response.status_code, 200) if not response.status_code == 200: return False @@ -187,37 +191,37 @@ def tearDown(self): def test_get_current_info(self,): data = self.stock.get_current_info(["YHOO","AAPL","GOOG"]) logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code,200) + self.assertEqual(data.status_code, 200) def test_get_news_feed(self,): data = self.stock.get_news_feed('YHOO') logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code,200) + self.assertEqual(data.status_code, 200) def test_get_historical_info_with_args(self,): data = self.stock.get_historical_info('YHOO',items=['Open','Close','High','Low'], limit=5,startDate='2014-09-11',endDate='2015-02-10') logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code,200) + self.assertEqual(data.status_code, 200) def test_get_historical_info_without_args(self,): data = self.stock.get_historical_info('YHOO') logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code,200) + self.assertEqual(data.status_code, 200) def test_get_options_info(self,): data = self.stock.get_options_info('YHOO') logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code,200) + self.assertEqual(data.status_code, 200) def test_get_index_summary(self,): data = self.stock.get_index_summary('GOOG',('Volume','Change')) logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code,200) + self.assertEqual(data.status_code, 200) def test_get_industry_index(self,): data = self.stock.get_industry_index(112) logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code,200) + self.assertEqual(data.status_code, 200) class TestTable(unittest.TestCase): From cae771abf3921edeef0b2a058fabb65dc7aa0d2a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 9 Jun 2015 05:05:33 +0200 Subject: [PATCH 439/582] test show tables added --- tests/tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 70f18be..3d16d19 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -46,7 +46,13 @@ def tearDown(self): def test_desc(self,): response = self.yql.desc('weather.forecast') - logging.debug(prettyfy(response,'json')) + logging.debug(prettyfy(response, 'json')) + self.assertEqual(response.status_code, 200) + + def test_show_tables(self,): + yql = MYQL(format='xml', community=False) + response = yql.showTables(format='xml') + logging.debug(prettyfy(response, 'xml')) self.assertEqual(response.status_code, 200) def test_use(self): From c54c5d1768b091da2ec79a83fcd5fbe441430de3 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 9 Jun 2015 05:09:36 +0200 Subject: [PATCH 440/582] cleaning the error house --- myql/errors.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/myql/errors.py b/myql/errors.py index 3c42acc..fd32288 100755 --- a/myql/errors.py +++ b/myql/errors.py @@ -8,30 +8,4 @@ def __init__(self, msg=None): def __str__(self): return repr(self.msg) - -class NoConfigFileError(Exception): - '''Error raised when the config file passed - doesn't exist - ''' - def __init__(self, msg=None): - if not msg: - msg = 'Config file is invalid or doesn\'t exist' - - self.msg = msg - - def __str__(self,): - return repr(self.msg) - - -class NoConfigParameter(Exception): - '''Error raised when config parameters don't exist - ''' - def __init__(self, msg=None): - if not msg: - msg = 'Invalid config parameters' - - self.msg = msg - - def __str__(self,): - return repr(self.msg) - + From 3a44190d6bab5e7b9887696e6b4c441bda66e5a2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 9 Jun 2015 05:14:41 +0200 Subject: [PATCH 441/582] show coverage reports after success --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0680d61..2c91b44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,5 @@ before_install: - openssl aes-256-cbc -K $encrypted_60f42691f4f2_key -iv $encrypted_60f42691f4f2_iv -in credentials.json.enc -out credentials.json -d after_success: + - coverage report - coveralls From fe7be3c176f0f7d49f07adeb5dce67a0147ce597 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 9 Jun 2015 06:50:23 +0200 Subject: [PATCH 442/582] #1 : restoring badges --- README.md | 6 ++++-- setup.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f1fc012..9d966dd 100755 --- a/README.md +++ b/README.md @@ -2,11 +2,13 @@ ========= [![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) - [![Code Issues](https://www.quantifiedcode.com/project/gh:josuebrunel:myql/badge.svg)](https://www.quantifiedcode.com/app/project/gh:josuebrunel:myql) +[![PyPI](https://img.shields.io/pypi/status/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/v/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/dm/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) -[![PyPI](https://img.shields.io/pypi/l/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) +[![PyPI](https://img.shields.io/pypi/pyversions/myql.svg)](https://pypi.python.org/pypi/myql) +[![PyPI](https://img.shields.io/pypi/implementation/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![Coverage Status](https://coveralls.io/repos/josuebrunel/myql/badge.svg?branch=testing)](https://coveralls.io/r/josuebrunel/myql?branch=testing) +[![PyPI](https://img.shields.io/pypi/l/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) mYQL is a Python wrapper of the Yahoo Query Language. **[Full Documentation](http://myql.readthedocs.org/en/latest/)** diff --git a/setup.py b/setup.py index 5e38a8b..362dbc8 100755 --- a/setup.py +++ b/setup.py @@ -25,8 +25,13 @@ def read(fname): packages = find_packages(), tests_suite="tests", classifiers = [ + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Development Status :: 5 - Production/Stable', From 74e50dea0fc34676c83760460ba21ef293e85e99 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 10 Jun 2015 07:42:33 +0200 Subject: [PATCH 443/582] #1: updating documentation --- README.md | 3 +++ docs/stockscraper.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d966dd..44f3bc6 100755 --- a/README.md +++ b/README.md @@ -401,6 +401,9 @@ Delete records >>> response = yql.getGUID('josue_brunel') # Deal with the response ``` +#### Stocks Scraper + +The full documentation on ***StockScraper*** is [here](https://myql.readthedocs.org/en/latest/stockscraper/) #### Release Notes diff --git a/docs/stockscraper.md b/docs/stockscraper.md index 8d89783..488d12e 100644 --- a/docs/stockscraper.md +++ b/docs/stockscraper.md @@ -11,7 +11,7 @@ Full [Documentation](http://www.gurchet-rai.net/dev/yahoo-finance-yql) * ***format*** : xml or json * ***debug*** : True or False -* ***oauth*** : YOAuth object +* ***oauth*** : yahoo_oauth (OAuth1 or OAuth2) ```python from myql.contrib.stockscraper import StockRetriever From 889a78ae200167450629d14ad003852a0dacc201 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 10 Jun 2015 07:43:34 +0200 Subject: [PATCH 444/582] #1: updating branch for coverage badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44f3bc6..08abf6d 100755 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![PyPI](https://img.shields.io/pypi/dm/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/pyversions/myql.svg)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/implementation/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) -[![Coverage Status](https://coveralls.io/repos/josuebrunel/myql/badge.svg?branch=testing)](https://coveralls.io/r/josuebrunel/myql?branch=testing) +[![Coverage Status](https://coveralls.io/repos/josuebrunel/myql/badge.svg?branch=testing)](https://coveralls.io/r/josuebrunel/myql?branch=master) [![PyPI](https://img.shields.io/pypi/l/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) From 57ff4062d78281b4aa2c70fffbf47ee459ce5422 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 12 Jun 2015 05:38:53 +0200 Subject: [PATCH 445/582] updating dev_requirements.txt for python3 --- dev_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index f087e44..b89d456 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -21,5 +21,4 @@ requests>=2.7.0 requests-oauthlib>=0.5.0 six>=1.9.0 tornado>=4.2 -wsgiref>=0.1.2 yahoo-oauth>=0.1.4 From 8d205618d07663d33fb0c8086b0b3ab2e03beb5f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 12 Jun 2015 05:41:07 +0200 Subject: [PATCH 446/582] no more pyflake8 --- dev_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index b89d456..a557ba1 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -8,7 +8,6 @@ click>=4.0 coverage>=3.7.1 coveralls>=0.5 docopt>=0.6.2 -flake8>=2.4.1 ghp-import>=0.4.1 livereload>=2.4.0 mccabe>=0.3 From 13318c0b88eb397bfbbee34d015daf12fa5c26a1 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 12 Jun 2015 05:50:20 +0200 Subject: [PATCH 447/582] finance namespace added --- myql/contrib/__init__.py | 4 +++- myql/contrib/finance/__init__.py | 3 +++ myql/contrib/finance/stockscraper/__init__.py | 1 + myql/contrib/{ => finance}/stockscraper/stockretriever.py | 0 myql/contrib/stockscraper/__init__.py | 1 - tests/tests.py | 2 +- 6 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 myql/contrib/finance/__init__.py create mode 100644 myql/contrib/finance/stockscraper/__init__.py rename myql/contrib/{ => finance}/stockscraper/stockretriever.py (100%) delete mode 100644 myql/contrib/stockscraper/__init__.py diff --git a/myql/contrib/__init__.py b/myql/contrib/__init__.py index 532bae6..cb782df 100755 --- a/myql/contrib/__init__.py +++ b/myql/contrib/__init__.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + from myql.contrib import table -from myql.contrib import stockscraper +from myql.contrib import finance diff --git a/myql/contrib/finance/__init__.py b/myql/contrib/finance/__init__.py new file mode 100644 index 0000000..763f32b --- /dev/null +++ b/myql/contrib/finance/__init__.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +from myql.contrib.finance import stockscraper diff --git a/myql/contrib/finance/stockscraper/__init__.py b/myql/contrib/finance/stockscraper/__init__.py new file mode 100644 index 0000000..f774907 --- /dev/null +++ b/myql/contrib/finance/stockscraper/__init__.py @@ -0,0 +1 @@ +from myql.contrib.finance.stockscraper.stockretriever import StockRetriever diff --git a/myql/contrib/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py similarity index 100% rename from myql/contrib/stockscraper/stockretriever.py rename to myql/contrib/finance/stockscraper/stockretriever.py diff --git a/myql/contrib/stockscraper/__init__.py b/myql/contrib/stockscraper/__init__.py deleted file mode 100644 index fa42c42..0000000 --- a/myql/contrib/stockscraper/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from myql.contrib.stockscraper.stockretriever import StockRetriever diff --git a/tests/tests.py b/tests/tests.py index 3d16d19..d75a0a0 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -16,7 +16,7 @@ from myql.contrib.table import BaseInput from myql.contrib.table import Binder, BinderFunction, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset -from myql.contrib.stockscraper import StockRetriever +from myql.contrib.finance.stockscraper import StockRetriever logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logging.getLogger('Test-mYQL') From acecb01b08ba6690b961dc3b28f027e33d27a71a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 13 Jun 2015 00:34:08 +0200 Subject: [PATCH 448/582] #123: Tests OK, method OK --- .../finance/stockscraper/stockretriever.py | 15 ++++++++++++++- tests/tests.py | 6 ++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index fdf8895..2608f8d 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -5,10 +5,12 @@ from __future__ import absolute_import -import calendar +import re import datetime from datetime import date, timedelta +import requests + from myql.myql import MYQL class StockRetriever(MYQL): @@ -63,3 +65,14 @@ def get_industry_index(self, index_id,items=[],format='json'): """ response = self.select('yahoo.finance.industry',items).where(['id','=',index_id]) return response + + def stock_lookup(self, name): + """Retrieves all symbols belonging to a company + """ + url = "http://autoc.finance.yahoo.com/autoc?query={0}&callback=YAHOO.Finance.SymbolSuggest.ssCallback".format(name) + + response = requests.get(url) + + json_data = re.match("YAHOO\.Finance\.SymbolSuggest.ssCallback\((.*)\)", response.text) + # return a bytes + return json_data.groups()[0].encode() diff --git a/tests/tests.py b/tests/tests.py index d75a0a0..14fd5b4 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -229,6 +229,12 @@ def test_get_industry_index(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) + def test_stock_lookup(self,): + data = self.stock.stock_lookup('Google') + logging.debug(pretty_json(data)) + #self.assertEqual(data.status_code, 200) + #self.assertIn(('GOOG','GOOGL','GOOG.MX'),) + class TestTable(unittest.TestCase): From 1224c6b55d956f82d48b0dc5d086ee1f56ed7705 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 13 Jun 2015 01:14:55 +0200 Subject: [PATCH 449/582] fix #123 : find symbols by company name OK --- .../finance/stockscraper/stockretriever.py | 20 +++++++++++++++++-- tests/tests.py | 6 +++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index 2608f8d..22106e9 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -6,6 +6,7 @@ from __future__ import absolute_import import re +import json import datetime from datetime import date, timedelta @@ -74,5 +75,20 @@ def stock_lookup(self, name): response = requests.get(url) json_data = re.match("YAHOO\.Finance\.SymbolSuggest.ssCallback\((.*)\)", response.text) - # return a bytes - return json_data.groups()[0].encode() + try: + json_data = json_data.groups()[0] + except: + json_data = '' + + return type('Response', (requests.Response,),{ + 'text' : json_data, + 'content': json_data.encode(), + 'status_code': response.status_code, + 'reason': response.reason, + 'encoding': response.encoding, + 'apparent_encoding': response.apparent_encoding, + 'cookies': response.cookies, + 'headers': response.headers, + 'json': lambda : json.loads(json_data), + 'url': response.url + }) diff --git a/tests/tests.py b/tests/tests.py index 14fd5b4..6d77ace 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import os +import pdb import logging import json import unittest @@ -231,9 +232,8 @@ def test_get_industry_index(self,): def test_stock_lookup(self,): data = self.stock.stock_lookup('Google') - logging.debug(pretty_json(data)) - #self.assertEqual(data.status_code, 200) - #self.assertIn(('GOOG','GOOGL','GOOG.MX'),) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) class TestTable(unittest.TestCase): From 1109d50496b7662953f5c71630d5808cf72bc1ac Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 14 Jun 2015 15:23:01 +0200 Subject: [PATCH 450/582] Changing coverage settings --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 33b21e0..88ee395 100644 --- a/.coveragerc +++ b/.coveragerc @@ -24,5 +24,5 @@ exclude_lines = ignore_errors = True [html] -directory = coverage_html_report +directory = html_report From ede8507e1f740ced471dddedb049cfb1e718a4ca Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 14 Jun 2015 15:26:24 +0200 Subject: [PATCH 451/582] YQL object created. MYQL and StockScraper subclassing this Object --- myql/contrib/finance/stockscraper/stockretriever.py | 4 ++-- myql/myql.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index 22106e9..0dab9bf 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -12,9 +12,9 @@ import requests -from myql.myql import MYQL +from myql.myql import YQL -class StockRetriever(MYQL): +class StockRetriever(YQL): def __init__(self, format='json', debug=False, oauth=None): """Initialize the object diff --git a/myql/myql.py b/myql/myql.py index f6e60e0..e95ca2d 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -10,7 +10,8 @@ logging.getLogger('requests').setLevel(logging.WARNING) -class MYQL(object): + +class YQL(object): '''Yet another Python Yahoo! Query Language Wrapper Attributes: - url : data provider url @@ -261,6 +262,14 @@ def where(self, *args): return response + +class MYQL(YQL): + + def __init__(self, *args, **kwargs): + + super(MYQL, self).__init__(**kwargs) + + ###################################################### # # HELPERS From 50e7624c7f4011d17a6bd4372ac17a49e0903990 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 14 Jun 2015 16:08:59 +0200 Subject: [PATCH 452/582] landscape.io : improving module health --- myql/contrib/finance/stockscraper/stockretriever.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index 0dab9bf..6e0619c 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -7,7 +7,6 @@ import re import json -import datetime from datetime import date, timedelta import requests @@ -49,19 +48,19 @@ def get_historical_info(self, symbol,items=None, startDate=None, endDate=None, l response = self.select('yahoo.finance.historicaldata',items,limit).where(['symbol','=',symbol],['startDate','=',startDate],['endDate','=',endDate]) return response - def get_options_info(self, symbol, items=[], expiration='', format=format): + def get_options_info(self, symbol, items=None, expiration='', format=format): """get_options_data() uses the yahoo.finance.options table to retrieve call and put options from the options page. """ response = self.select('yahoo.finance.options',items).where(['symbol','=',symbol],[] if not expiration else ['expiration','=',expiration]) return response - def get_index_summary(self, symbol, items=[],format='json'): + def get_index_summary(self, symbol, items=None,format='json'): """ """ response = self.select('yahoo.finance.quoteslist',items).where(['symbol','=',symbol]) return response - def get_industry_index(self, index_id,items=[],format='json'): + def get_industry_index(self, index_id,items=None,format='json'): """retrieves all symbols that belong to an industry. """ response = self.select('yahoo.finance.industry',items).where(['id','=',index_id]) From 94146756b304a8772659a7db54bdd6c5abf44c70 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 14 Jun 2015 16:11:11 +0200 Subject: [PATCH 453/582] #1: health badge added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 08abf6d..4c8443c 100755 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ ========= [![Build Status](https://travis-ci.org/josuebrunel/myql.svg?branch=master)](https://travis-ci.org/josuebrunel/myql) [![Documentation Status](https://readthedocs.org/projects/myql/badge/?version=latest)](https://myql.readthedocs.org) +[![Code Health](https://landscape.io/github/josuebrunel/myql/master/landscape.svg?style=flat)](https://landscape.io/github/josuebrunel/myql/master) [![PyPI](https://img.shields.io/pypi/status/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/v/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) [![PyPI](https://img.shields.io/pypi/dm/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) From f6fd5e6aa9a932cb7881afd3b6ff8c0aabf69dd2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 14 Jun 2015 16:17:24 +0200 Subject: [PATCH 454/582] fixing errors manager import --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index e95ca2d..da60957 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -2,7 +2,7 @@ import logging import requests -import myql.errors +from myql.myql import errors logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") From 0864c22714e179f13de2d1c6f5afa161d58c724b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 14 Jun 2015 16:26:08 +0200 Subject: [PATCH 455/582] #126: trying to improve health --- myql/contrib/table/base.py | 2 +- myql/myql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index 4d1ea01..4bcb8e5 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -165,7 +165,7 @@ def removeInput(self, key_id, input_type='key'): def addPaging(self,paging): """Add paging to Binder """ - if not self.paging: + if not vars(self).get('paging', None): self.paging = paging root = self.etree diff --git a/myql/myql.py b/myql/myql.py index da60957..ef652e6 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -2,7 +2,7 @@ import logging import requests -from myql.myql import errors +from myql import errors logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") From e186a7b12233da91632eae00a6d4d6f2358aafab Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 06:11:45 +0200 Subject: [PATCH 456/582] #104: test_get_weather_in OK --- tests/__init__.py | 1 + tests/tests.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index af963fb..281ee45 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from tests.tests import TestMYQL +from tests.tests import TestWeather from tests.tests import TestStockScraper from tests.tests import TestTable from tests.tests import TestOAuth diff --git a/tests/tests.py b/tests/tests.py index 6d77ace..9a878e8 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -17,6 +17,7 @@ from myql.contrib.table import BaseInput from myql.contrib.table import Binder, BinderFunction, InputKey, InputValue, PagingPage, PagingUrl, PagingOffset +from myql.contrib.weather import Weather from myql.contrib.finance.stockscraper import StockRetriever logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") @@ -187,6 +188,17 @@ def test_yahoo_fantasy_sport(self,): print(current_team['team_id'],current_team['name'],current_team['number_of_trades'],current_team['number_of_moves']) +class TestWeather(unittest.TestCase): + """Weather module unit test + """ + def setUp(self,): + self.weather = Weather(format='json') + + def test_get_weather_in(self): + data = self.weather.get_weather_in('choisy-le-roi') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + class TestStockScraper(unittest.TestCase): def setUp(self,): From 8e677ea6f4de4acb6cc0d9fe7a184c18bb762a1f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 06:12:16 +0200 Subject: [PATCH 457/582] #104: get_weather_in place OK --- myql/contrib/__init__.py | 1 + myql/contrib/weather/__init__.py | 3 +++ myql/contrib/weather/weather.py | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 myql/contrib/weather/__init__.py create mode 100644 myql/contrib/weather/weather.py diff --git a/myql/contrib/__init__.py b/myql/contrib/__init__.py index cb782df..7a36d06 100755 --- a/myql/contrib/__init__.py +++ b/myql/contrib/__init__.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +from myql.contrib import weather from myql.contrib import table from myql.contrib import finance diff --git a/myql/contrib/weather/__init__.py b/myql/contrib/weather/__init__.py new file mode 100644 index 0000000..afdd8f4 --- /dev/null +++ b/myql/contrib/weather/__init__.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +from myql.contrib.weather.weather import Weather diff --git a/myql/contrib/weather/weather.py b/myql/contrib/weather/weather.py new file mode 100644 index 0000000..848dc70 --- /dev/null +++ b/myql/contrib/weather/weather.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import + +from myql.myql import YQL + + +class Weather(YQL): + """Weather Class + """ + + def __init__(self, *args, **kwargs): + + super(Weather, self).__init__(**kwargs) + + + def get_weather_in(self, place): + """Return weather info according to place + """ + response = self.select('weather.forecast').where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)]) + return response From 4ac8cae974f5a5d62d743f3d311bddc97eac6a00 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 06:34:54 +0200 Subject: [PATCH 458/582] #104: test for get_weather_in with items and unit added --- tests/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 9a878e8..f38484c 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -199,6 +199,12 @@ def test_get_weather_in(self): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) + def test_get_weather_in_with_unit(self): + data = self.weather.get_weather_in('choisy-le-roi', 'c',['location', 'units', 'item.condition']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + class TestStockScraper(unittest.TestCase): def setUp(self,): From 80bd4fde1ac37b521be3944ab6deae44a329758c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 06:35:19 +0200 Subject: [PATCH 459/582] #104: get_weather_in with items and unit OK --- myql/contrib/weather/weather.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/myql/contrib/weather/weather.py b/myql/contrib/weather/weather.py index 848dc70..249c513 100644 --- a/myql/contrib/weather/weather.py +++ b/myql/contrib/weather/weather.py @@ -7,13 +7,17 @@ class Weather(YQL): """Weather Class """ - def __init__(self, *args, **kwargs): - + def __init__(self, unit=None, **kwargs): + """Initialize a weather object + """ super(Weather, self).__init__(**kwargs) + self.unit = unit + - def get_weather_in(self, place): + def get_weather_in(self, place, unit=None, items=None): """Return weather info according to place """ - response = self.select('weather.forecast').where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)]) + unit = unit if unit else self.unit + response = self.select('weather.forecast', items=items).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)], ['u','=',unit] if unit else []) return response From d36b87d840669da1b9cc895f330447333f772f48 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 06:46:51 +0200 Subject: [PATCH 460/582] #104: get_weather_forecast OK --- myql/contrib/weather/weather.py | 9 +++++++++ tests/tests.py | 9 +++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/myql/contrib/weather/weather.py b/myql/contrib/weather/weather.py index 249c513..ae66e1d 100644 --- a/myql/contrib/weather/weather.py +++ b/myql/contrib/weather/weather.py @@ -10,6 +10,8 @@ class Weather(YQL): def __init__(self, unit=None, **kwargs): """Initialize a weather object """ + kwargs.update({'community':False}) + super(Weather, self).__init__(**kwargs) self.unit = unit @@ -21,3 +23,10 @@ def get_weather_in(self, place, unit=None, items=None): unit = unit if unit else self.unit response = self.select('weather.forecast', items=items).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)], ['u','=',unit] if unit else []) return response + + def get_weather_forecast(self, place, unit=None): + """Return weather forecast accoriding to place + """ + unit = unit if unit else self.unit + response = self.select('weather.forecast', items=['item.forecast']).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)], ['u','=',unit] if unit else []) + return response diff --git a/tests/tests.py b/tests/tests.py index f38484c..ddbf895 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -192,7 +192,7 @@ class TestWeather(unittest.TestCase): """Weather module unit test """ def setUp(self,): - self.weather = Weather(format='json') + self.weather = Weather(unit='c', format='json') def test_get_weather_in(self): data = self.weather.get_weather_in('choisy-le-roi') @@ -203,7 +203,12 @@ def test_get_weather_in_with_unit(self): data = self.weather.get_weather_in('choisy-le-roi', 'c',['location', 'units', 'item.condition']) logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) - + + def test_get_weather_forecast(self,): + data = self.weather.get_weather_forecast('choisy-le-roi') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + class TestStockScraper(unittest.TestCase): From aaeabf284203012061726d86f9c9a63990f50774 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 07:29:24 +0200 Subject: [PATCH 461/582] #104: get_current_condition and get_weather_description OK --- myql/contrib/weather/weather.py | 13 ++++++++++++- tests/tests.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/myql/contrib/weather/weather.py b/myql/contrib/weather/weather.py index ae66e1d..41774ff 100644 --- a/myql/contrib/weather/weather.py +++ b/myql/contrib/weather/weather.py @@ -16,7 +16,6 @@ def __init__(self, unit=None, **kwargs): self.unit = unit - def get_weather_in(self, place, unit=None, items=None): """Return weather info according to place """ @@ -30,3 +29,15 @@ def get_weather_forecast(self, place, unit=None): unit = unit if unit else self.unit response = self.select('weather.forecast', items=['item.forecast']).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)], ['u','=',unit] if unit else []) return response + + def get_weather_description(self, place): + """Return weather description + """ + response = self.select('weather.forecast', items=['item.condition.text']).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)]) + return response + + def get_current_condition(self, place): + """Return weather condition + """ + response = self.select('weather.forecast', items=['item.condition']).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)]) + return response diff --git a/tests/tests.py b/tests/tests.py index ddbf895..233d325 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -209,6 +209,16 @@ def test_get_weather_forecast(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) + def test_get_weather_description(self,): + data = self.weather.get_weather_description('dolisie') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + def test_get_current_condition(self,): + data = self.weather.get_current_condition('caracas') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + class TestStockScraper(unittest.TestCase): From d3664e4a46ce91fda605869f29aa79c6a90f1957 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 07:46:06 +0200 Subject: [PATCH 462/582] fix #104 : weather module OK --- myql/contrib/weather/weather.py | 24 +++++++++++++++++++++--- tests/tests.py | 17 ++++++++++++++++- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/myql/contrib/weather/weather.py b/myql/contrib/weather/weather.py index 41774ff..38961e0 100644 --- a/myql/contrib/weather/weather.py +++ b/myql/contrib/weather/weather.py @@ -27,17 +27,35 @@ def get_weather_forecast(self, place, unit=None): """Return weather forecast accoriding to place """ unit = unit if unit else self.unit - response = self.select('weather.forecast', items=['item.forecast']).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)], ['u','=',unit] if unit else []) + response = self.get_weather_in(place, items=['item.forecast'], unit=unit) return response def get_weather_description(self, place): """Return weather description """ - response = self.select('weather.forecast', items=['item.condition.text']).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)]) + response = self.get_weather_in(place, items=['item.condition.text']) return response def get_current_condition(self, place): """Return weather condition """ - response = self.select('weather.forecast', items=['item.condition']).where(['woeid','IN',('SELECT woeid FROM geo.places WHERE text="{0}"'.format(place),)]) + response = self.get_weather_in(place, items=['item.condition']) + return response + + def get_current_atmosphere(self, place): + """Return weather atmosphere + """ + response = self.get_weather_in(place, items=['atmosphere']) + return response + + def get_current_wind(self, place): + """Return weather wind + """ + response = self.get_weather_in(place, items=['wind']) + return response + + def get_astronomy(self, place): + """Return sunrise and sunset time + """ + response = self.get_weather_in(place, items=['astronomy']) return response diff --git a/tests/tests.py b/tests/tests.py index 233d325..03cc841 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -215,7 +215,22 @@ def test_get_weather_description(self,): self.assertEqual(data.status_code, 200) def test_get_current_condition(self,): - data = self.weather.get_current_condition('caracas') + data = self.weather.get_current_condition('Nantes') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + def test_get_current_atmosphere(self,): + data = self.weather.get_current_atmosphere('Scotland') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + def test_get_current_wind(self,): + data = self.weather.get_current_wind('Barcelona') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + def test_get_astronomy(self,): + data = self.weather.get_astronomy('Congo') logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) From 622297b32391a49ff858ae224e5bcb3e939d0bb5 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 07:57:00 +0200 Subject: [PATCH 463/582] #126 : improving library health --- myql/contrib/finance/stockscraper/stockretriever.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index 6e0619c..d9beaf7 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -21,21 +21,21 @@ def __init__(self, format='json', debug=False, oauth=None): super(StockRetriever, self).__init__(community=True, format=format, debug=debug, oauth=oauth) - def get_current_info(self, symbolList, columns=None, format='json'): + def get_current_info(self, symbolList, columns=None): """get_current_info() uses the yahoo.finance.quotes datatable to get all of the stock information presented in the main table on a typical stock page and a bunch of data from the key statistics page. """ response = self.select('yahoo.finance.quotes',columns).where(['symbol','in',symbolList]) return response - def get_news_feed(self, symbol,format='json'): + def get_news_feed(self, symbol): """get_news_feed() uses the rss data table to get rss feeds under the Headlines and Financial Blogs headings on a typical stock page. """ rss_url='http://finance.yahoo.com/rss/headline?s={0}'.format(symbol) response = self.select('rss',['title','link','description'],limit=2).where(['url','=',rss_url]) return response - def get_historical_info(self, symbol,items=None, startDate=None, endDate=None, limit=None, format='json'): + def get_historical_info(self, symbol,items=None, startDate=None, endDate=None, limit=None): """get_historical_info() uses the csv datatable to retrieve all available historical data on a typical historical prices page """ today = date.today() @@ -48,19 +48,19 @@ def get_historical_info(self, symbol,items=None, startDate=None, endDate=None, l response = self.select('yahoo.finance.historicaldata',items,limit).where(['symbol','=',symbol],['startDate','=',startDate],['endDate','=',endDate]) return response - def get_options_info(self, symbol, items=None, expiration='', format=format): + def get_options_info(self, symbol, items=None, expiration=''): """get_options_data() uses the yahoo.finance.options table to retrieve call and put options from the options page. """ response = self.select('yahoo.finance.options',items).where(['symbol','=',symbol],[] if not expiration else ['expiration','=',expiration]) return response - def get_index_summary(self, symbol, items=None,format='json'): + def get_index_summary(self, symbol, items=None): """ """ response = self.select('yahoo.finance.quoteslist',items).where(['symbol','=',symbol]) return response - def get_industry_index(self, index_id,items=None,format='json'): + def get_industry_index(self, index_id,items=None): """retrieves all symbols that belong to an industry. """ response = self.select('yahoo.finance.industry',items).where(['id','=',index_id]) From a1f66312f8e5cfb6ce7ae8a582894daf6c3eb0c4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 08:10:53 +0200 Subject: [PATCH 464/582] #126 : improving lib health --- myql/myql.py | 507 ++++++++++++++++++++++++++------------------------- 1 file changed, 254 insertions(+), 253 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index ef652e6..bffe01c 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -12,286 +12,287 @@ class YQL(object): - '''Yet another Python Yahoo! Query Language Wrapper - Attributes: - - url : data provider url - - table : default table, so you won't have to specify a table later - - format : default format of the responses - - diagnostics : set to to see diagnostics on queries - - community : set to to have access to community tables - ''' - public_url = 'https://query.yahooapis.com/v1/public/yql' - private_url = 'http://query.yahooapis.com/v1/yql' - community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table + '''Yet another Python Yahoo! Query Language Wrapper + Attributes: + - url : data provider url + - table : default table, so you won't have to specify a table later + - format : default format of the responses + - diagnostics : set to to see diagnostics on queries + - community : set to to have access to community tables + ''' + public_url = 'https://query.yahooapis.com/v1/public/yql' + private_url = 'http://query.yahooapis.com/v1/yql' + community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table - def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): - self.community = community # True means access to community data - self.format = format - self._table = None - self._query = None # used to build query when using methods such as , , ... + self._payload = {} # Last payload + self.diagnostics = False # Who knows, someone would like to turn it ON lol + self.limit = None + self.crossProduct = crossProduct + self.jsonCompact = jsonCompact + self.debug = debug - if oauth: - self.oauth = oauth + if oauth: + self.oauth = oauth - def __repr__(self): - '''Returns information on the current instance - ''' - return ": {0} - : {1} ".format(self.community, self.format) + def __repr__(self): + '''Returns information on the current instance + ''' + return ": {0} - : {1} ".format(self.community, self.format) - def payloadBuilder(self, query, format=None): - '''Build the payload''' - if self.community : - query = self.community_data + query # access to community data tables + def payloadBuilder(self, query, format=None): + '''Build the payload''' + if self.community : + query = self.community_data + query # access to community data tables - if vars(self).get('yql_table_url') : # Attribute only defined when MYQL.use has been called before - query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query + if vars(self).get('yql_table_url') : # Attribute only defined when MYQL.use has been called before + query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query - self._query = query - logger.info("QUERY = {0}".format(query)) + self._query = query + logger.info("QUERY = {0}".format(query)) - payload = { - 'q' : query, - 'callback' : '', #This is not javascript - 'diagnostics' : self.diagnostics, - 'format' : format if format else self.format, - 'debug': self.debug, - 'jsonCompact': self.jsonCompact - } - if self.crossProduct: - payload['crossProduct'] = self.crossProduct + payload = { + 'q' : query, + 'callback' : '', #This is not javascript + 'diagnostics' : self.diagnostics, + 'format' : format if format else self.format, + 'debug': self.debug, + 'jsonCompact': self.jsonCompact + } + if self.crossProduct: + payload['crossProduct'] = self.crossProduct - self._payload = payload - logger.info("PAYLOAD = {0}".format(payload)) - - return payload - - def rawQuery(self, query, format=None, pretty=False): - '''Executes a YQL query and returns a response - >>>... - >>> resp = yql.rawQuery('select * from weather.forecast where woeid=2502265') - >>> - ''' - if format: - format = format - else: - format = self.format + self._payload = payload + logger.info("PAYLOAD = {0}".format(payload)) + + return payload + + def rawQuery(self, query, format=None, pretty=False): + '''Executes a YQL query and returns a response + >>>... + >>> resp = yql.rawQuery('select * from weather.forecast where woeid=2502265') + >>> + ''' + if format: + format = format + else: + format = self.format - payload = self.payloadBuilder(query, format=format) - response = self.executeQuery(payload) - if pretty : - response = self.buildResponse(response) - - return response - - def executeQuery(self, payload): - '''Execute the query and returns and response''' - if vars(self).get('oauth'): - if not self.oauth.token_is_valid(): # Refresh token if token has expired - self.oauth.refresh_token() - response = self.oauth.session.get(self.private_url, params= payload, header_auth=True) - else: - response = requests.get(self.public_url, params= payload) - - self._response = response # Saving last response object. - return response - - def clauseFormatter(self, cond): - '''Formats conditions - args is a list of ['column', 'operator', 'value'] - ''' - - if cond[1].lower() == 'in': - if len(cond[2]) > 1: - cond[2] = "({0})".format(','.join(map(str,[ "'{0}'".format(e) for e in cond[2] ]))) - else: - cond[2] = "({0})".format(','.join(map(str,[ "{0}".format(e) for e in cond[2] ]))) + payload = self.payloadBuilder(query, format=format) + response = self.executeQuery(payload) + if pretty : + response = self.buildResponse(response) + + return response + + def executeQuery(self, payload): + '''Execute the query and returns and response''' + if vars(self).get('oauth'): + if not self.oauth.token_is_valid(): # Refresh token if token has expired + self.oauth.refresh_token() + response = self.oauth.session.get(self.private_url, params= payload, header_auth=True) + else: + response = requests.get(self.public_url, params= payload) + + self._response = response # Saving last response object. + return response + + def clauseFormatter(self, cond): + '''Formats conditions + args is a list of ['column', 'operator', 'value'] + ''' + + if cond[1].lower() == 'in': + if len(cond[2]) > 1: + cond[2] = "({0})".format(','.join(map(str,[ "'{0}'".format(e) for e in cond[2] ]))) + else: + cond[2] = "({0})".format(','.join(map(str,[ "{0}".format(e) for e in cond[2] ]))) - cond = " ".join(cond) - else: - cond[2] = "'{0}'".format(cond[2]) - cond = ''.join(cond) + cond = " ".join(cond) + else: + cond[2] = "'{0}'".format(cond[2]) + cond = ''.join(cond) - return cond + return cond - def buildResponse(self, response): - '''Try to return a pretty formatted response object - ''' - try: - r = response.json() - result = r['query']['results'] - response = { - 'num_result': r['query']['count'] , - 'result': result - } - except (Exception,) as e: - print(e) - return response.content - return response - - - ###################################################### - # - # MAIN METHODS - # - ##################################################### - - ##USE - def use(self, url, name='mytable'): - '''Changes the data provider - >>> yql.use('http://myserver.com/mytables.xml') - ''' - self.yql_table_url = url - self.yql_table_name = name - return {'table url': url, 'table name': name} - - ##DESC - def desc(self, table=None): - '''Returns table description - >>> yql.desc('geo.countries') - >>> - ''' - if not table: - #query = "desc {0} ".format(self._table) - raise errors.NoTableSelectedError('No table selected') - query = "desc {0}".format(table) - response = self.rawQuery(query) - - return response - - ##GET - def get(self, table=None, items=[], limit=''): - '''Just a select which returns a response - >>> yql.get("geo.countries', ['name', 'woeid'], 5") - ''' - self._table = table - if not items: - items = ['*'] - self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) - if limit: - self._query += "limit {0}".format(limit) - - if not self._table : - raise errors.NoTableSelectedError('Please select a table') + def buildResponse(self, response): + '''Try to return a pretty formatted response object + ''' + try: + r = response.json() + result = r['query']['results'] + response = { + 'num_result': r['query']['count'] , + 'result': result + } + except (Exception,) as e: + print(e) + return response.content + + return response + + + ###################################################### + # + # MAIN METHODS + # + ##################################################### + + ##USE + def use(self, url, name='mytable'): + '''Changes the data provider + >>> yql.use('http://myserver.com/mytables.xml') + ''' + self.yql_table_url = url + self.yql_table_name = name + return {'table url': url, 'table name': name} + + ##DESC + def desc(self, table=None): + '''Returns table description + >>> yql.desc('geo.countries') + >>> + ''' + if not table: + #query = "desc {0} ".format(self._table) + raise errors.NoTableSelectedError('No table selected') + query = "desc {0}".format(table) + response = self.rawQuery(query) + + return response + + ##GET + def get(self, table=None, items=[], limit=''): + '''Just a select which returns a response + >>> yql.get("geo.countries', ['name', 'woeid'], 5") + ''' + self._table = table + if not items: + items = ['*'] + self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) + if limit: + self._query += "limit {0}".format(limit) + + if not self._table : + raise errors.NoTableSelectedError('Please select a table') - payload = self.payloadBuilder(self._query) - response = self.executeQuery(payload) + payload = self.payloadBuilder(self._query) + response = self.executeQuery(payload) - return response + return response - ## SELECT - def select(self, table=None, items=[], limit=''): - '''This method simulate a select on a table - >>> yql.select('geo.countries', limit=5) - >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) - ''' - self._table = table - if not items: - items = ['*'] - self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) - try: #Checking wether a limit is set or not - self._limit = limit - except (Exception,) as e: - pass - - return self - - ## INSERT - def insert(self, table,items, values): - """This method allows to insert data into table - >>> yql.insert('bi.ly.shorten',('login','apiKey','longUrl'),('YOUR LOGIN','YOUR API KEY','YOUR LONG URL')) - """ - values = ["'{0}'".format(e) for e in values] - self._query = "INSERT INTO {0} ({1}) VALUES ({2})".format(table,','.join(items),','.join(values)) - payload = self.payloadBuilder(self._query) - response = self.executeQuery(payload) - - return response - - ## UPDATE - def update(self, table, items, values): - """Updates a YQL Table - >>> yql.update('yql.storage',['value'],['https://josuebrunel.orkg']).where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) - """ - self._table = table - self._limit = None - items_values = ','.join(["{0} = '{1}'".format(k,v) for k,v in zip(items,values)]) - self._query = "UPDATE {0} SET {1}".format(self._table, items_values) - - return self - - ## DELETE - def delete(self, table): - """Deletes record in table - >>> yql.delete('yql.storage').where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) - """ - self._table = table - self._limit = None - self._query = "DELETE FROM {0}".format(self._table) - - return self - - ## WHERE - def where(self, *args): - ''' This method simulates a where condition. Use as follow: - >>> yql.select('mytable').where(['name', '=', 'alain'], ['location', '!=', 'paris']) - ''' - if not self._table: - raise errors.NoTableSelectedError('No Table Selected') - - clause = [] - self._query += ' WHERE ' - for x in args: - if x: - x = self.clauseFormatter(x) - clause.append(x) - - self._query += ' AND '.join(clause) - - if self._limit : - self._query += " LIMIT {0}".format(self._limit) - - payload = self.payloadBuilder(self._query) - response = self.executeQuery(payload) - - return response + ## SELECT + def select(self, table=None, items=[], limit=''): + '''This method simulate a select on a table + >>> yql.select('geo.countries', limit=5) + >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) + ''' + self._table = table + if not items: + items = ['*'] + self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) + try: #Checking wether a limit is set or not + self._limit = limit + except (Exception,) as e: + pass + + return self + + ## INSERT + def insert(self, table,items, values): + """This method allows to insert data into table + >>> yql.insert('bi.ly.shorten',('login','apiKey','longUrl'),('YOUR LOGIN','YOUR API KEY','YOUR LONG URL')) + """ + values = ["'{0}'".format(e) for e in values] + self._query = "INSERT INTO {0} ({1}) VALUES ({2})".format(table,','.join(items),','.join(values)) + payload = self.payloadBuilder(self._query) + response = self.executeQuery(payload) + + return response + + ## UPDATE + def update(self, table, items, values): + """Updates a YQL Table + >>> yql.update('yql.storage',['value'],['https://josuebrunel.orkg']).where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) + """ + self._table = table + self._limit = None + items_values = ','.join(["{0} = '{1}'".format(k,v) for k,v in zip(items,values)]) + self._query = "UPDATE {0} SET {1}".format(self._table, items_values) + + return self + + ## DELETE + def delete(self, table): + """Deletes record in table + >>> yql.delete('yql.storage').where(['name','=','store://YEl70PraLLMSMuYAauqNc7']) + """ + self._table = table + self._limit = None + self._query = "DELETE FROM {0}".format(self._table) + + return self + + ## WHERE + def where(self, *args): + ''' This method simulates a where condition. Use as follow: + >>> yql.select('mytable').where(['name', '=', 'alain'], ['location', '!=', 'paris']) + ''' + if not self._table: + raise errors.NoTableSelectedError('No Table Selected') + + clause = [] + self._query += ' WHERE ' + for x in args: + if x: + x = self.clauseFormatter(x) + clause.append(x) + + self._query += ' AND '.join(clause) + + if self._limit : + self._query += " LIMIT {0}".format(self._limit) + + payload = self.payloadBuilder(self._query) + response = self.executeQuery(payload) + + return response class MYQL(YQL): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): - super(MYQL, self).__init__(**kwargs) + super(MYQL, self).__init__(**kwargs) - ###################################################### - # - # HELPERS - # - ##################################################### + ###################################################### + # + # HELPERS + # + ##################################################### - def getGUID(self, username): - '''Returns the guid of the username provided - >>> guid = self.getGUID('josue_brunel') - >>> guid - ''' - response = self.select('yahoo.identity').where(['yid', '=', username]) + def getGUID(self, username): + '''Returns the guid of the username provided + >>> guid = self.getGUID('josue_brunel') + >>> guid + ''' + response = self.select('yahoo.identity').where(['yid', '=', username]) - return response + return response - def showTables(self, format='json'): - '''Return list of all available tables''' + def showTables(self, format='json'): + '''Return list of all available tables''' - query = 'SHOW TABLES' - payload = self.payloadBuilder(query, format) + query = 'SHOW TABLES' + payload = self.payloadBuilder(query, format) - response = self.executeQuery(payload) + response = self.executeQuery(payload) - return response + return response From 57db0918bff319937463d1397e8257ea95a21193 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 08:25:06 +0200 Subject: [PATCH 465/582] #126: improving library health --- myql/errors.py | 16 ++++++++-------- myql/myql.py | 19 +++++++++++-------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/myql/errors.py b/myql/errors.py index fd32288..92258e5 100755 --- a/myql/errors.py +++ b/myql/errors.py @@ -1,11 +1,11 @@ class NoTableSelectedError(Exception): - '''Error raised when no table has been selected - ''' - def __init__(self, msg=None): - if not msg: - msg = 'No table selected' - self.msg = msg + '''Error raised when no table has been selected + ''' + def __init__(self, msg=None): + if not msg: + msg = 'No table selected' + self.msg = msg - def __str__(self): - return repr(self.msg) + def __str__(self): + return repr(self.msg) diff --git a/myql/myql.py b/myql/myql.py index bffe01c..165b6e6 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -53,7 +53,7 @@ def payloadBuilder(self, query, format=None): query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query self._query = query - logger.info("QUERY = {0}".format(query)) + logger.info("QUERY = %s" %(query,)) payload = { 'q' : query, @@ -67,7 +67,7 @@ def payloadBuilder(self, query, format=None): payload['crossProduct'] = self.crossProduct self._payload = payload - logger.info("PAYLOAD = {0}".format(payload)) + logger.info("PAYLOAD = %s " %(payload,)) return payload @@ -166,7 +166,7 @@ def desc(self, table=None): return response ##GET - def get(self, table=None, items=[], limit=''): + def get(self, table=None, items=None, limit=''): '''Just a select which returns a response >>> yql.get("geo.countries', ['name', 'woeid'], 5") ''' @@ -187,7 +187,7 @@ def get(self, table=None, items=[], limit=''): ## SELECT - def select(self, table=None, items=[], limit=''): + def select(self, table=None, items=None, limit=None): '''This method simulate a select on a table >>> yql.select('geo.countries', limit=5) >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) @@ -196,10 +196,13 @@ def select(self, table=None, items=[], limit=''): if not items: items = ['*'] self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) - try: #Checking wether a limit is set or not - self._limit = limit - except (Exception,) as e: - pass + + self._limit = limit + + # try: #Checking wether a limit is set or not + # self._limit = limit + # except (Exception,) as e: + # pass return self From 0ef859eecc073455a6c3e443c53a4b5816f39a0e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 16 Jun 2015 08:44:11 +0200 Subject: [PATCH 466/582] #126: improving lib health --- myql/contrib/finance/stockscraper/stockretriever.py | 3 ++- myql/contrib/table/base.py | 5 +++-- myql/contrib/table/binder.py | 5 ++--- myql/contrib/table/table.py | 2 +- myql/myql.py | 5 ----- myql/utils.py | 2 +- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index d9beaf7..2d4f49e 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -76,7 +76,8 @@ def stock_lookup(self, name): json_data = re.match("YAHOO\.Finance\.SymbolSuggest.ssCallback\((.*)\)", response.text) try: json_data = json_data.groups()[0] - except: + except (Execption,) as e: + print(e) json_data = '' return type('Response', (requests.Response,),{ diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index 4bcb8e5..d04ebff 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -90,7 +90,7 @@ def addUrl(self, url): """Add url to binder """ - if not url in self.urls: + if url not in self.urls: self.urls.append(url) root = self.etree @@ -126,7 +126,7 @@ def removeUrl(self, url): def addInput(self, key): """Add key to input : key, value or map """ - if not key in self.inputs: + if key not in self.inputs: self.inputs.append(key) root = self.etree @@ -216,6 +216,7 @@ def __init__(self, input_type, id, type, paramType, like='', required=False, def self.id = id self.like = like # as is a python , that's why in argument we use self.type = type + self.paramType = paramType self.required = required self.default = default self.private = private diff --git a/myql/contrib/table/binder.py b/myql/contrib/table/binder.py index daf1619..cc5e251 100755 --- a/myql/contrib/table/binder.py +++ b/myql/contrib/table/binder.py @@ -1,12 +1,11 @@ from myql.contrib.table.base import Base, BaseInput, BasePaging, BaseBinder -from xml.etree import cElementTree as xtree class Binder(BaseBinder): """Represent a binder : , , , diff --git a/myql/contrib/table/table.py b/myql/contrib/table/table.py index 679a07f..45671b9 100755 --- a/myql/contrib/table/table.py +++ b/myql/contrib/table/table.py @@ -3,7 +3,7 @@ from xml.etree import cElementTree as xtree from myql.contrib.table.base import Base -from myql.contrib.table.binder import BinderMeta, Binder, BinderFunction +from myql.contrib.table.binder import Binder, BinderFunction class Table(Base): """Class representating a YQL Table From 616c7fcc077eaead8e3f9ed86f70329d047dfd42 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 18 Jun 2015 08:23:09 +0200 Subject: [PATCH 468/582] stock_lookup changed into get_symbols --- myql/contrib/finance/stockscraper/stockretriever.py | 4 ++-- tests/tests.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index e2cf12e..fdf5d2f 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -66,7 +66,7 @@ def get_industry_index(self, index_id,items=None): response = self.select('yahoo.finance.industry',items).where(['id','=',index_id]) return response - def stock_lookup(self, name): + def get_symbols(self, name): """Retrieves all symbols belonging to a company """ url = "http://autoc.finance.yahoo.com/autoc?query={0}&callback=YAHOO.Finance.SymbolSuggest.ssCallback".format(name) @@ -80,7 +80,7 @@ def stock_lookup(self, name): print(e) json_data = '' - return type('Response', (requests.Response,),{ + return type('response', (requests.Response,),{ 'text' : json_data, 'content': json_data.encode(), 'status_code': response.status_code, diff --git a/tests/tests.py b/tests/tests.py index 03cc841..87ccb0b 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -278,8 +278,8 @@ def test_get_industry_index(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) - def test_stock_lookup(self,): - data = self.stock.stock_lookup('Google') + def test_get_symbols(self,): + data = self.stock.get_symbols('Google') logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) From ca1cecebf0c0f6ea5adc22054b19a50f972feb8a Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 18 Jun 2015 09:17:05 +0200 Subject: [PATCH 469/582] improving stockscraper --- docs/stockscraper.md | 116 ++++++++++++++++++++++++++++++++++++++++--- mkdocs.yml | 1 + myql/myql.py | 28 +++++------ 3 files changed, 123 insertions(+), 22 deletions(-) diff --git a/docs/stockscraper.md b/docs/stockscraper.md index 488d12e..8bea0af 100644 --- a/docs/stockscraper.md +++ b/docs/stockscraper.md @@ -11,10 +11,10 @@ Full [Documentation](http://www.gurchet-rai.net/dev/yahoo-finance-yql) * ***format*** : xml or json * ***debug*** : True or False -* ***oauth*** : yahoo_oauth (OAuth1 or OAuth2) +* ***oauth*** : yahoo_oauth (OAuth1) ```python -from myql.contrib.stockscraper import StockRetriever +from myql.contrib.finance.stockscraper import StockRetriever stocks = StockRetriever(format='json') ``` @@ -26,7 +26,7 @@ stocks = StockRetriever(format='json') * ***columns*** : List of column to fetch ```python -from myql.contrib.stockscraper import StockRetriever +from myql.contrib.finance.stockscraper import StockRetriever stocks = StockRetriever(format='json') data = stocks.get_current_info(["YHOO","AAPL","GOOG"]) ``` @@ -136,7 +136,7 @@ data = stocks.get_current_info(["YHOO","AAPL","GOOG"]) * ***symbol*** : Symbol news to retrieve ```python -from myql.contrib.stockscraper import StockRetriever +from myql.contrib.finance.stockscraper import StockRetriever stocks = StockRetriever(format='json') data = stocks.get_news_feed('YHOO') ``` @@ -174,7 +174,7 @@ data = stocks.get_news_feed('YHOO') * ***limit*** : number of results to return ```python -from myql.contrib.stockscraper import StockRetriever +from myql.contrib.finance.stockscraper import StockRetriever stocks = StockRetriever(format='json') data = stocks.get_historical_info('YHOO',items=['Open','Close','High','Low'], limit=5,startDate='2014-09-11',endDate='2015-02-10') ``` @@ -231,7 +231,7 @@ data = stocks.get_historical_info('YHOO',items=['Open','Close','High','Low'], li * ***expiration*** : Date of expiration (type : str) ```python -from myql.contrib.stockscraper import StockRetriever +from myql.contrib.finance.stockscraper import StockRetriever stocks = StockRetriever(format='json') data = stocks.get_options_info('YHOO') ``` @@ -257,7 +257,7 @@ data = stocks.get_options_info('YHOO') * ***items*** : list of attributes to retrieve ```python -from myql.contrib.stockscraper import StockRetriever +from myql.contrib.finance.stockscraper import StockRetriever stocks = StockRetriever(format='json') data = stocks.get_index_summary('GOOG',('Volume','Change')) ``` @@ -284,7 +284,7 @@ data = stocks.get_index_summary('GOOG',('Volume','Change')) * ***items*** : list of attributes to retrieve ```python -from myql.contrib.stockscraper import StockRetriever +from myql.contrib.finance.stockscraper import StockRetriever stocks = StockRetriever(format='json') data = stocks.get_industry_index(112) ``` @@ -336,3 +336,103 @@ data = stocks.get_industry_index(112) } ``` +#### *StockRetriever.get_symbols(company_name)* + +     **Always returns data as JSON** + +```python +from myql.contrib.finance.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_symbols('Google') +``` + +```json +{ + "ResultSet": { + "Query": "google", + "Result": [ + { + "exch": "NMS", + "exchDisp": "NASDAQ", + "name": "Google Inc.", + "symbol": "GOOG", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "NMS", + "exchDisp": "NASDAQ", + "name": "Google Inc.", + "symbol": "GOOGL", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "GER", + "exchDisp": "XETRA", + "name": "Google Inc.", + "symbol": "GGQ7.DE", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "MEX", + "exchDisp": "Mexico", + "name": "Google Inc.", + "symbol": "GOOG.MX", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "MEX", + "exchDisp": "Mexico", + "name": "GOOGLE-A", + "symbol": "GOOGL.MX", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "BUE", + "exchDisp": "Buenos Aires", + "name": "Google Inc.", + "symbol": "GOOGL.BA", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "FRA", + "exchDisp": "Frankfurt", + "name": "GOOGLE-A", + "symbol": "GGQ1.F", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "MUN", + "exchDisp": "Munich", + "name": "GOOGLE-A", + "symbol": "GGQ1.MU", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "EBS", + "exchDisp": "Swiss", + "name": "GOOGLE-A", + "symbol": "GOOGL.SW", + "type": "S", + "typeDisp": "Equity" + }, + { + "exch": "MUN", + "exchDisp": "Munich", + "name": "GOOGLE-C", + "symbol": "GGQ7.MU", + "type": "S", + "typeDisp": "Equity" + } + ] + } +} + +``` diff --git a/mkdocs.yml b/mkdocs.yml index e84cdc5..83ef098 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,6 +3,7 @@ repo_url: https://github.com/josuebrunel/myql site_url: http://myql.readthedocs.org/en/latest/myql/ site_description: MYQL documentation site_author: Josue Kouka +use_directory_urls: True google_analytics: ['UA-32441224-4', 'myql.readthedocs.org'] pages: - Home : index.md diff --git a/myql/myql.py b/myql/myql.py index a08e49e..fc37b68 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -54,20 +54,20 @@ def payloadBuilder(self, query, format=None): self._query = query logger.info("QUERY = %s" %(query,)) - + payload = { - 'q' : query, - 'callback' : '', #This is not javascript - 'diagnostics' : self.diagnostics, - 'format' : format if format else self.format, + 'q': query, + 'callback': '',#This is not javascript + 'diagnostics': self.diagnostics, + 'format': format if format else self.format, 'debug': self.debug, 'jsonCompact': self.jsonCompact } if self.crossProduct: payload['crossProduct'] = self.crossProduct - + self._payload = payload - logger.info("PAYLOAD = %s " %(payload,)) + logger.info("PAYLOAD = %s " %(payload, )) return payload @@ -75,23 +75,23 @@ def rawQuery(self, query, format=None, pretty=False): '''Executes a YQL query and returns a response >>>... >>> resp = yql.rawQuery('select * from weather.forecast where woeid=2502265') - >>> + >>> ''' if format: format = format else: format = self.format - + payload = self.payloadBuilder(query, format=format) response = self.executeQuery(payload) - if pretty : + if pretty: response = self.buildResponse(response) return response def executeQuery(self, payload): '''Execute the query and returns and response''' - if vars(self).get('oauth'): + if vars(self).get('oauth'): if not self.oauth.token_is_valid(): # Refresh token if token has expired self.oauth.refresh_token() response = self.oauth.session.get(self.private_url, params= payload, header_auth=True) @@ -108,10 +108,10 @@ def clauseFormatter(self, cond): if cond[1].lower() == 'in': if len(cond[2]) > 1: - cond[2] = "({0})".format(','.join(map(str,[ "'{0}'".format(e) for e in cond[2] ]))) + cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]]))) else: - cond[2] = "({0})".format(','.join(map(str,[ "{0}".format(e) for e in cond[2] ]))) - + cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]]))) + cond = " ".join(cond) else: cond[2] = "'{0}'".format(cond[2]) From 590fa46dbfa81cc1e717ff46b961b4a65c70fa37 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 26 Jun 2015 22:32:07 +0200 Subject: [PATCH 470/582] yahoo finance get xchange added and OK --- myql/contrib/finance/stockscraper/stockretriever.py | 7 +++++++ myql/myql.py | 3 +++ tests/tests.py | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index fdf5d2f..e64735b 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -66,6 +66,13 @@ def get_industry_index(self, index_id,items=None): response = self.select('yahoo.finance.industry',items).where(['id','=',index_id]) return response + def get_xchange(self, pairs, items=None): + """Retrieves currency exchange rate data for given pair(s). + Accepts both where pair='eurusd, gbpusd' and where pair in ('eurusd', 'gpbusd, usdaud') + """ + response = self.select('yahoo.finance.xchange', items).where(['pair', 'in', pairs]) + return response + def get_symbols(self, name): """Retrieves all symbols belonging to a company """ diff --git a/myql/myql.py b/myql/myql.py index fc37b68..6c83af6 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -1,3 +1,6 @@ +"""Simple Python Wrapper of the Yahoo! Query Language +""" + from __future__ import absolute_import import logging diff --git a/tests/tests.py b/tests/tests.py index 87ccb0b..c55d9f1 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -283,6 +283,11 @@ def test_get_symbols(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) + def test_get_xchange(self,): + data = self.stock.get_xchange(['EURUSD','GBPUSD']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + class TestTable(unittest.TestCase): From 3a6efcf8fb36fa249cff86add4cf53dc13be0d3f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 26 Jun 2015 22:57:44 +0200 Subject: [PATCH 471/582] get_dividendhistory OK --- .../finance/stockscraper/stockretriever.py | 26 ++++++++++++++----- tests/tests.py | 5 ++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index e64735b..222abea 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -19,7 +19,18 @@ def __init__(self, format='json', debug=False, oauth=None): """Initialize the object """ super(StockRetriever, self).__init__(community=True, format=format, debug=debug, oauth=oauth) + + def __get_time_range(self, startDate, endDate): + """Return time range + """ + today = date.today() + start_date = today - timedelta(days=today.weekday(), weeks=1) + end_date = start_date + timedelta(days=4) + startDate = startDate if startDate else str(start_date) + endDate = endDate if endDate else str(end_date) + + return startDate, endDate def get_current_info(self, symbolList, columns=None): """get_current_info() uses the yahoo.finance.quotes datatable to get all of the stock information presented in the main table on a typical stock page @@ -38,13 +49,7 @@ def get_news_feed(self, symbol): def get_historical_info(self, symbol,items=None, startDate=None, endDate=None, limit=None): """get_historical_info() uses the csv datatable to retrieve all available historical data on a typical historical prices page """ - today = date.today() - start_date = today - timedelta(days=today.weekday(), weeks=1) - end_date = start_date + timedelta(days=4) - - startDate = startDate if startDate else str(start_date) - endDate = endDate if endDate else str(end_date) - + startDate, endDate = self.__get_time_range(startDate, endDate) response = self.select('yahoo.finance.historicaldata',items,limit).where(['symbol','=',symbol],['startDate','=',startDate],['endDate','=',endDate]) return response @@ -73,6 +78,13 @@ def get_xchange(self, pairs, items=None): response = self.select('yahoo.finance.xchange', items).where(['pair', 'in', pairs]) return response + def get_dividendhistory(self, symbol, startDate, endDate, items=None): + """Retrieves divident history + """ + startDate, endDate = self.__get_time_range(startDate, endDate) + response = self.select('yahoo.finance.dividendhistory', items).where(['symbol', '=', symbol], ['startDate', '=', startDate], ['endDate', '=', endDate]) + return response + def get_symbols(self, name): """Retrieves all symbols belonging to a company """ diff --git a/tests/tests.py b/tests/tests.py index c55d9f1..a2909a8 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -288,6 +288,11 @@ def test_get_xchange(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) + def test_get_dividendhistory(self,): + data = self.stock.get_dividendhistory('AAPL',"2008-01-01", "2015-06-15") + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + class TestTable(unittest.TestCase): From 95bbb9496fec6e6c7237f8176516599bedbeba2e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Fri, 26 Jun 2015 23:00:15 +0200 Subject: [PATCH 472/582] just fixing something --- myql/contrib/weather/weather.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/contrib/weather/weather.py b/myql/contrib/weather/weather.py index 38961e0..0c7fce8 100644 --- a/myql/contrib/weather/weather.py +++ b/myql/contrib/weather/weather.py @@ -10,7 +10,7 @@ class Weather(YQL): def __init__(self, unit=None, **kwargs): """Initialize a weather object """ - kwargs.update({'community':False}) + kwargs.update({'community': False}) super(Weather, self).__init__(**kwargs) From 985be59c2f1bfc2e08116f694a252d8c2464e482 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 27 Jun 2015 15:51:24 +0200 Subject: [PATCH 473/582] #1: updating stockscraper module --- docs/stockscraper.md | 124 ++++++++++++++++++ .../finance/stockscraper/stockretriever.py | 2 +- tests/tests.py | 4 +- 3 files changed, 127 insertions(+), 3 deletions(-) diff --git a/docs/stockscraper.md b/docs/stockscraper.md index 8bea0af..a326af0 100644 --- a/docs/stockscraper.md +++ b/docs/stockscraper.md @@ -336,6 +336,130 @@ data = stocks.get_industry_index(112) } ``` +#### *StockRetriever.get_xchange_rate(pairs, items=None)* + +```python +from myql.contrib.finance.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_xchange_rate(['EURUSD','GBPUSD']) +``` + +```json +{ + "query": { + "count": 2, + "created": "2015-06-27T13:48:51Z", + "lang": "en-US", + "results": { + "rate": [ + { + "Ask": "1.1174", + "Bid": "1.1162", + "Date": "6/27/2015", + "Name": "EUR/USD", + "Rate": "1.1168", + "Time": "12:53pm", + "id": "EURUSD" + }, + { + "Ask": "1.5756", + "Bid": "1.5738", + "Date": "6/27/2015", + "Name": "GBP/USD", + "Rate": "1.5747", + "Time": "12:53pm", + "id": "GBPUSD" + } + ] + } + } +} + +``` + +#### *StockRetriever.get_dividendhistory(symbol, startDate, endDate)* + +```python +from myql.contrib.finance.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = stocks.get_dividendhistory('AAPL',"2008-01-01", "2015-06-15") +``` + +```json +{ + "query": { + "count": 12, + "created": "2015-06-27T13:42:27Z", + "lang": "en-US", + "results": { + "quote": [ + { + "Date": "2015-05-07", + "Dividends": "0.520000", + "Symbol": "AAPL" + }, + { + "Date": "2015-02-05", + "Dividends": "0.470000", + "Symbol": "AAPL" + }, + { + "Date": "2014-11-06", + "Dividends": "0.470000", + "Symbol": "AAPL" + }, + { + "Date": "2014-08-07", + "Dividends": "0.470000", + "Symbol": "AAPL" + }, + { + "Date": "2014-05-08", + "Dividends": "0.470000", + "Symbol": "AAPL" + }, + { + "Date": "2014-02-06", + "Dividends": "0.435710", + "Symbol": "AAPL" + }, + { + "Date": "2013-11-06", + "Dividends": "0.435710", + "Symbol": "AAPL" + }, + { + "Date": "2013-08-08", + "Dividends": "0.435710", + "Symbol": "AAPL" + }, + { + "Date": "2013-05-09", + "Dividends": "0.435710", + "Symbol": "AAPL" + }, + { + "Date": "2013-02-07", + "Dividends": "0.378570", + "Symbol": "AAPL" + }, + { + "Date": "2012-11-07", + "Dividends": "0.378570", + "Symbol": "AAPL" + }, + { + "Date": "2012-08-09", + "Dividends": "0.378570", + "Symbol": "AAPL" + } + ] + } + } +} + +``` + #### *StockRetriever.get_symbols(company_name)*      **Always returns data as JSON** diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index 222abea..3069a07 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -71,7 +71,7 @@ def get_industry_index(self, index_id,items=None): response = self.select('yahoo.finance.industry',items).where(['id','=',index_id]) return response - def get_xchange(self, pairs, items=None): + def get_xchange_rate(self, pairs, items=None): """Retrieves currency exchange rate data for given pair(s). Accepts both where pair='eurusd, gbpusd' and where pair in ('eurusd', 'gpbusd, usdaud') """ diff --git a/tests/tests.py b/tests/tests.py index a2909a8..ff37c90 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -283,8 +283,8 @@ def test_get_symbols(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) - def test_get_xchange(self,): - data = self.stock.get_xchange(['EURUSD','GBPUSD']) + def test_get_xchange_rate(self,): + data = self.stock.get_xchange_rate(['EURUSD','GBPUSD']) logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) From 21c319822aae6235e1584440a544bd7deaa8880c Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 27 Jun 2015 15:51:42 +0200 Subject: [PATCH 474/582] update test script --- run_tests.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 65fd2f4..91aaf45 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -10,6 +10,5 @@ else Test='' fi -python -m unittest tests$TestCase$Test - - +coverage run --source=myql -m unittest tests$TestCase$Test +coverage report From 4decc198c3b1aa15febecfefa9af0e691c406a72 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 30 Jun 2015 17:43:01 +0200 Subject: [PATCH 475/582] #131: test added and seems OK --- tests/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index ff37c90..3e20d31 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -244,7 +244,12 @@ def tearDown(self): pass def test_get_current_info(self,): - data = self.stock.get_current_info(["YHOO","AAPL","GOOG"]) + data = self.stock.get_current_info(["YHOO","AAPL","GOOG","J&KBANK.BO"]) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + def test_get_current_info_with_one_symbol(self,): + data = self.stock.get_current_info(["J&KBANK.BO"]) logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) From cad7d9a53b44a926095c23e4a36846e80d2d3492 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 30 Jun 2015 17:43:36 +0200 Subject: [PATCH 476/582] fix #131: handling of one symbol OK --- myql/contrib/finance/stockscraper/stockretriever.py | 1 + 1 file changed, 1 insertion(+) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index 3069a07..2f0f4a5 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -36,6 +36,7 @@ def get_current_info(self, symbolList, columns=None): """get_current_info() uses the yahoo.finance.quotes datatable to get all of the stock information presented in the main table on a typical stock page and a bunch of data from the key statistics page. """ + #symbolList = symbolList if not isinstance(symbolList, str) else (symbolList,) response = self.select('yahoo.finance.quotes',columns).where(['symbol','in',symbolList]) return response From 191354ffb02a79d17b1afae7efabef5642816354 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Tue, 30 Jun 2015 18:06:21 +0200 Subject: [PATCH 477/582] fix #131: handling of one symbol OK --- myql/myql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 6c83af6..8ca0d96 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -108,9 +108,9 @@ def clauseFormatter(self, cond): '''Formats conditions args is a list of ['column', 'operator', 'value'] ''' - if cond[1].lower() == 'in': - if len(cond[2]) > 1: + #if len(cond[2]) > 1: + if not isinstance(cond[2], str): cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]]))) else: cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]]))) From 35dd1fdc5bcd38c6e5cc8bc4e123cd88f2ac2842 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 5 Jul 2015 23:49:37 +0200 Subject: [PATCH 478/582] generates coverage html report after running tests --- run_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run_tests.sh b/run_tests.sh index 91aaf45..0175f75 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -12,3 +12,4 @@ fi coverage run --source=myql -m unittest tests$TestCase$Test coverage report +coverage html From 042f20674a67f6582a6d7ef301aa7702641439fb Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 5 Jul 2015 23:50:32 +0200 Subject: [PATCH 479/582] #126 : improving library health by cleaning the base house --- myql/contrib/table/base.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/myql/contrib/table/base.py b/myql/contrib/table/base.py index d04ebff..cd7e99e 100644 --- a/myql/contrib/table/base.py +++ b/myql/contrib/table/base.py @@ -2,27 +2,6 @@ class Base(object): - def addElement(self, elt, tagName=None): - root = self.etree - t_elt = root.find(tagName) - if not t_elt: - t_elt = ctree.SubElement(root, tagName) - - root.append(t_elt) - return True - - def removeElement(self, elt, tagName=None): - root = self.etree - t_elt = root.find(tagName) - - try: - root.remove(t_elt) - return True - except (Exception,) as e: - print(e) - - return False - def addFunction(self, func_code, from_file=''): """Add function """ From 5a52d3fe8b2a60f437515a8284f98e0166691dd4 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sun, 5 Jul 2015 23:51:33 +0200 Subject: [PATCH 480/582] fix #133: special treatment for IN cond with SELECT statement in it --- myql/myql.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 8ca0d96..4a64547 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -109,9 +109,10 @@ def clauseFormatter(self, cond): args is a list of ['column', 'operator', 'value'] ''' if cond[1].lower() == 'in': - #if len(cond[2]) > 1: - if not isinstance(cond[2], str): + if not isinstance(cond[2], str) and 'select' not in cond[2][0].lower() : cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]]))) + elif not isinstance(cond[2], str) and 'select' in cond[2][0].lower() : + cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]]))) else: cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]]))) From bd759c6713777fec597606a90bfab06d57e95c8b Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Jul 2015 09:38:37 +0200 Subject: [PATCH 481/582] Additional info added about v1.2.4 --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c8443c..b7fc40f 100755 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ [![PyPI](https://img.shields.io/pypi/l/myql.svg?style=flat)](https://pypi.python.org/pypi/myql) -mYQL is a Python wrapper of the Yahoo Query Language. **[Full Documentation](http://myql.readthedocs.org/en/latest/)** +mYQL is a Python wrapper of the Yahoo Query Language. Yahoo! Query Language Documentation and Support =============================================== @@ -408,8 +408,14 @@ The full documentation on ***StockScraper*** is [here](https://myql.readthedocs. #### Release Notes +##### 1.2.4 +----------- + +* Weather module added +* StockScraper now under Finance namespace + ##### 1.2.3 -------- +----------- * Fixed issue related to date in StockRetriver.get_historical_info [#107](https://github.com/josuebrunel/myql/issues/107) * Fixed issue with **IN** condition in **where** clause [#106](https://github.com/josuebrunel/myql/issues/107) From 34484ec8e068d44b765dd8b512697d25f92daf6e Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Jul 2015 09:40:34 +0200 Subject: [PATCH 482/582] removing useless function --- myql/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/myql/utils.py b/myql/utils.py index 18c9bb5..0732be8 100644 --- a/myql/utils.py +++ b/myql/utils.py @@ -24,11 +24,3 @@ def prettyfy(response, format='json'): else: return pretty_xml(response.content) - -def dump(response): - """Print a pretty formatted response content - """ - try: - print(pretty_json(response.content)) - except (Exception, ) : - print(pretty_xml(response.content)) From 1b8301db31e7db2625e35b0f592e4934456f398f Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Jul 2015 10:46:54 +0200 Subject: [PATCH 483/582] #1 : updating doc: weather documentation OK --- docs/weather.md | 492 ++++++++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 3 +- 2 files changed, 494 insertions(+), 1 deletion(-) create mode 100644 docs/weather.md diff --git a/docs/weather.md b/docs/weather.md new file mode 100644 index 0000000..efef5c5 --- /dev/null +++ b/docs/weather.md @@ -0,0 +1,492 @@ +Weater +======= + +### **Definition** + +####*Weather(unit=None, **kwargs**)* + +* ***unit*** : alternative to default is **c** +* ***kwargs*** : Any possible argument of **YQL** + +```python +>>> from myql.contrib.weather import Weather +>>> weather = Weather(unit='c', format='json') +``` + + +### **Methods** + +#### *Weather.get_weather_in(place, unit=None, items=None)* +>Return weather information according to the place passed in + +```python +>>> data = weather.get_weather_in('choisy-le-roi', 'c',['location', 'units', 'item.condition']) +``` + +```json +{ + "query": { + "count": 1, + "created": "2015-07-08T08:34:05Z", + "lang": "en-US", + "results": { + "channel": { + "astronomy": { + "sunrise": "5:55 am", + "sunset": "9:53 pm" + }, + "atmosphere": { + "humidity": "59", + "pressure": "1015.92", + "rising": "0", + "visibility": "9.99" + }, + "description": "Yahoo! Weather for Choisy-le-Roi, FR", + "image": { + "height": "18", + "link": "http://weather.yahoo.com", + "title": "Yahoo! Weather", + "url": "http://l.yimg.com/a/i/brand/purplelogo//uh/us/news-wea.gif", + "width": "142" + }, + "item": { + "condition": { + "code": "30", + "date": "Wed, 08 Jul 2015 10:00 am CEST", + "temp": "18", + "text": "Partly Cloudy" + }, + "description": "\n
\nCurrent Conditions:
\nPartly Cloudy, 18 C
\n
Forecast:
\nWed - Cloudy. High: 22 Low: 13
\nThu - Partly Cloudy. High: 25 Low: 11
\nFri - Sunny. High: 28 Low: 13
\nSat - Mostly Sunny. High: 31 Low: 13
\nSun - Partly Cloudy. High: 25 Low: 15
\n
\nFull Forecast at Yahoo! Weather

\n(provided by The Weather Channel)
\n", + "forecast": [ + { + "code": "26", + "date": "8 Jul 2015", + "day": "Wed", + "high": "22", + "low": "13", + "text": "Cloudy" + }, + { + "code": "30", + "date": "9 Jul 2015", + "day": "Thu", + "high": "25", + "low": "11", + "text": "Partly Cloudy" + }, + { + "code": "32", + "date": "10 Jul 2015", + "day": "Fri", + "high": "28", + "low": "13", + "text": "Sunny" + }, + { + "code": "34", + "date": "11 Jul 2015", + "day": "Sat", + "high": "31", + "low": "13", + "text": "Mostly Sunny" + }, + { + "code": "30", + "date": "12 Jul 2015", + "day": "Sun", + "high": "25", + "low": "15", + "text": "Partly Cloudy" + } + ], + "guid": { + "content": "FRXX3747_2015_07_12_7_00_CEST", + "isPermaLink": "false" + }, + "lat": "48.76", + "link": "http://us.rd.yahoo.com/dailynews/rss/weather/Choisy-le-Roi__FR/*http://weather.yahoo.com/forecast/FRXX3747_c.html", + "long": "2.42", + "pubDate": "Wed, 08 Jul 2015 10:00 am CEST", + "title": "Conditions for Choisy-le-Roi, FR at 10:00 am CEST" + }, + "language": "en-us", + "lastBuildDate": "Wed, 08 Jul 2015 10:00 am CEST", + "link": "http://us.rd.yahoo.com/dailynews/rss/weather/Choisy-le-Roi__FR/*http://weather.yahoo.com/forecast/FRXX3747_c.html", + "location": { + "city": "Choisy-le-Roi", + "country": "France", + "region": "" + }, + "title": "Yahoo! Weather - Choisy-le-Roi, FR", + "ttl": "60", + "units": { + "distance": "km", + "pressure": "mb", + "speed": "km/h", + "temperature": "C" + }, + "wind": { + "chill": "18", + "direction": "270", + "speed": "25.75" + } + } + } + } +} + +``` + +#### *Weather.get_weather_forecast(place, unit=None)* +>Return weather forecast + +```python +>>> data = weather.get_weather_description('dolisie') +``` + +```json +{ + "query": { + "count": 1, + "created": "2015-07-08T08:35:27Z", + "lang": "en-US", + "results": { + "channel": { + "item": { + "condition": { + "text": "Cloudy" + } + } + } + } + } +} + +``` + +#### *Weather.get_current_condition(place)* +>get_current_condition + +```python +``` + +```json +``` + +#### *Weather.get_current_atmosphere(place)* +>Return weather atmosphere + +```python +>>> data = weather.get_current_condition('Nantes') +``` + +```json +{ + "count": 4, + "created": "2015-07-08T08:38:17Z", + "lang": "en-US", + "results": { + "channel": [ + { + "item": { + "condition": { + "code": "28", + "date": "Wed, 08 Jul 2015 10:00 am CEST", + "temp": "18", + "text": "Mostly Cloudy" + } + } + }, + { + "item": { + "condition": { + "code": "3200", + "date": "Wed, 08 Jul 2015 4:00 am EDT", + "temp": "19", + "text": "Unknown" + } + } + }, + { + "item": { + "condition": { + "code": "27", + "date": "Wed, 08 Jul 2015 5:01 am BRT", + "temp": "16", + "text": "Mostly Cloudy" + } + } + }, + { + "item": { + "condition": { + "code": "28", + "date": "Wed, 08 Jul 2015 7:58 am WEST", + "temp": "18", + "text": "Mostly Cloudy" + } + } + } + ] + } + } +} + +``` + +#### *Weather.get_current_atmosphere(place)* +>Return sunrise and sunset time + +```python +>>> data = weather.get_current_atmosphere('Scotland') +``` + +```json +{ + "query": { + "count": 10, + "created": "2015-07-08T08:43:26Z", + "lang": "en-US", + "results": { + "channel": [ + { + "atmosphere": { + "humidity": "95", + "pressure": "1008.7", + "rising": "1", + "visibility": "" + } + }, + { + "atmosphere": { + "humidity": "62", + "pressure": "1013.3", + "rising": "0", + "visibility": "16.09" + } + }, + { + "atmosphere": { + "humidity": "80", + "pressure": "1017.7", + "rising": "2", + "visibility": "9.66" + } + }, + { + "atmosphere": { + "humidity": "93", + "pressure": "1012.9", + "rising": "2", + "visibility": "8.05" + } + }, + { + "atmosphere": { + "humidity": "82", + "pressure": "1016.8", + "rising": "1", + "visibility": "" + } + }, + { + "atmosphere": { + "humidity": "99", + "pressure": "1015.92", + "rising": "0", + "visibility": "16.09" + } + }, + { + "atmosphere": { + "humidity": "89", + "pressure": "982.05", + "rising": "0", + "visibility": "16.09" + } + }, + { + "atmosphere": { + "humidity": "88", + "pressure": "1015.92", + "rising": "0", + "visibility": "16.09" + } + }, + { + "atmosphere": { + "humidity": "100", + "pressure": "1015.92", + "rising": "1", + "visibility": "8.05" + } + }, + { + "atmosphere": { + "humidity": "81", + "pressure": "1015.5", + "rising": "0", + "visibility": "16.09" + } + } + ] + } + } +} + +``` + +#### *Weather.get_current_wind(place)* +>Return weather wind + +```python +>>> data = weather.get_current_wind('Barcelona') +``` + +```json +{ + "query": { + "count": 7, + "created": "2015-07-08T08:44:49Z", + "lang": "en-US", + "results": { + "channel": [ + { + "wind": { + "chill": "27", + "direction": "110", + "speed": "14.48" + } + }, + { + "wind": { + "chill": "15", + "direction": "300", + "speed": "24.14" + } + }, + { + "wind": { + "chill": "23", + "direction": "", + "speed": "" + } + }, + { + "wind": { + "chill": "22", + "direction": "190", + "speed": "9.66" + } + }, + { + "wind": { + "chill": "31", + "direction": "220", + "speed": "17.7" + } + }, + { + "wind": { + "chill": "17", + "direction": "360", + "speed": "12.87" + } + }, + { + "wind": { + "chill": "26", + "direction": "80", + "speed": "28.97" + } + } + ] + } + } +} + +``` + +#### *Weather.get_astronomy(place)* +>Return sunrise and sunset time + +```python +>>> data = weather.get_astronomy('Congo') +``` + +```json +{ + "query": { + "count": 10, + "created": "2015-07-08T08:45:44Z", + "lang": "en-US", + "results": { + "channel": [ + { + "astronomy": { + "sunrise": "6:10 am", + "sunset": "6:11 pm" + } + }, + { + "astronomy": { + "sunrise": "7:07 am", + "sunset": "4:58 pm" + } + }, + { + "astronomy": { + "sunrise": "6:03 am", + "sunset": "6:41 pm" + } + }, + { + "astronomy": { + "sunrise": "5:38 am", + "sunset": "8:32 pm" + } + }, + { + "astronomy": { + "sunrise": "5:55 am", + "sunset": "8:27 pm" + } + }, + { + "astronomy": { + "sunrise": "5:55 am", + "sunset": "5:30 pm" + } + }, + { + "astronomy": { + "sunrise": "5:38 am", + "sunset": "5:20 pm" + } + }, + { + "astronomy": { + "sunrise": "6:05 am", + "sunset": "8:57 pm" + } + }, + { + "astronomy": { + "sunrise": "5:36 am", + "sunset": "7:57 pm" + } + }, + { + "astronomy": { + "sunrise": "5:55 am", + "sunset": "8:51 pm" + } + } + ] + } + } +} + +``` + + diff --git a/mkdocs.yml b/mkdocs.yml index 83ef098..914563d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,8 +8,9 @@ google_analytics: ['UA-32441224-4', 'myql.readthedocs.org'] pages: - Home : index.md - mYQL : myql.md -- StockScraper : stockscraper.md - Yahoo-OAuth : oauth.md +- StockScraper : stockscraper.md +- Weather : weather.md - Open Tables : table.md - Contribute : contrib.md theme: readthedocs From cd6613daa8c8927c046c232994f19482c03b2a43 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Jul 2015 11:01:59 +0200 Subject: [PATCH 484/582] get balancsheet method added --- myql/contrib/finance/stockscraper/stockretriever.py | 7 ++++++- tests/tests.py | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/myql/contrib/finance/stockscraper/stockretriever.py b/myql/contrib/finance/stockscraper/stockretriever.py index 2f0f4a5..47eeb46 100644 --- a/myql/contrib/finance/stockscraper/stockretriever.py +++ b/myql/contrib/finance/stockscraper/stockretriever.py @@ -36,7 +36,6 @@ def get_current_info(self, symbolList, columns=None): """get_current_info() uses the yahoo.finance.quotes datatable to get all of the stock information presented in the main table on a typical stock page and a bunch of data from the key statistics page. """ - #symbolList = symbolList if not isinstance(symbolList, str) else (symbolList,) response = self.select('yahoo.finance.quotes',columns).where(['symbol','in',symbolList]) return response @@ -86,6 +85,12 @@ def get_dividendhistory(self, symbol, startDate, endDate, items=None): response = self.select('yahoo.finance.dividendhistory', items).where(['symbol', '=', symbol], ['startDate', '=', startDate], ['endDate', '=', endDate]) return response + def get_balancesheet(self, symbol): + """Retrieves balance sheet + """ + response = self.select('yahoo.finance.balancesheet').where(['symbol', '=', symbol]) + return response + def get_symbols(self, name): """Retrieves all symbols belonging to a company """ diff --git a/tests/tests.py b/tests/tests.py index 3e20d31..69a4bc2 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -298,6 +298,11 @@ def test_get_dividendhistory(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) + def test_get_balancesheet(self,): + data = self.stock.get_balancesheet('YHOO') + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + class TestTable(unittest.TestCase): From ee274e6cf0d1e7787d74dcb834b327de4d8a56da Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Jul 2015 11:02:15 +0200 Subject: [PATCH 485/582] updating lib version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 362dbc8..f957bcf 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ __author__ = 'Josue Kouka' __email__ = 'josuebrunel@gmail.com' -__version__ = "1.2.3" +__version__ = "1.2.4" #requirements.txt with open('requirements.txt') as f: From 720f33fb5ee9667e995ba2d0d800a7c14316893d Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Jul 2015 11:09:04 +0200 Subject: [PATCH 486/582] #1: updating docs --- docs/stockscraper.md | 24 ++++++++++++++++++++++++ docs/weather.md | 25 ++++++++----------------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/docs/stockscraper.md b/docs/stockscraper.md index a326af0..bcf96af 100644 --- a/docs/stockscraper.md +++ b/docs/stockscraper.md @@ -460,6 +460,30 @@ data = stocks.get_dividendhistory('AAPL',"2008-01-01", "2015-06-15") ``` +#### *StockRetriever.get_balancesheet(symbol)* + +```python +from myql.contrib.finance.stockscraper import StockRetriever +stocks = StockRetriever(format='json') +data = self.stock.get_balancesheet('YHOO') +``` + +```json +{ + "query": { + "count": 1, + "created": "2015-07-08T09:01:12Z", + "lang": "en-US", + "results": { + "balancesheet": { + "symbol": "YHOO", + "timeframe": "quarterly" + } + } + } +} +``` + #### *StockRetriever.get_symbols(company_name)*      **Always returns data as JSON** diff --git a/docs/weather.md b/docs/weather.md index efef5c5..2c0dd81 100644 --- a/docs/weather.md +++ b/docs/weather.md @@ -9,8 +9,8 @@ Weater * ***kwargs*** : Any possible argument of **YQL** ```python ->>> from myql.contrib.weather import Weather ->>> weather = Weather(unit='c', format='json') +from myql.contrib.weather import Weather +weather = Weather(unit='c', format='json') ``` @@ -20,7 +20,7 @@ Weater >Return weather information according to the place passed in ```python ->>> data = weather.get_weather_in('choisy-le-roi', 'c',['location', 'units', 'item.condition']) +data = weather.get_weather_in('choisy-le-roi', 'c',['location', 'units', 'item.condition']) ``` ```json @@ -141,7 +141,7 @@ Weater >Return weather forecast ```python ->>> data = weather.get_weather_description('dolisie') +data = weather.get_weather_description('dolisie') ``` ```json @@ -164,20 +164,11 @@ Weater ``` -#### *Weather.get_current_condition(place)* ->get_current_condition - -```python -``` - -```json -``` - #### *Weather.get_current_atmosphere(place)* >Return weather atmosphere ```python ->>> data = weather.get_current_condition('Nantes') +data = weather.get_current_condition('Nantes') ``` ```json @@ -238,7 +229,7 @@ Weater >Return sunrise and sunset time ```python ->>> data = weather.get_current_atmosphere('Scotland') +data = weather.get_current_atmosphere('Scotland') ``` ```json @@ -340,7 +331,7 @@ Weater >Return weather wind ```python ->>> data = weather.get_current_wind('Barcelona') +data = weather.get_current_wind('Barcelona') ``` ```json @@ -411,7 +402,7 @@ Weater >Return sunrise and sunset time ```python ->>> data = weather.get_astronomy('Congo') +data = weather.get_astronomy('Congo') ``` ```json From 1d68eabcbeaca299b2d3bfe957ab45ce0fee9319 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Wed, 8 Jul 2015 14:57:57 +0200 Subject: [PATCH 487/582] remove use_directory_urls --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 914563d..3f2f882 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,6 @@ repo_url: https://github.com/josuebrunel/myql site_url: http://myql.readthedocs.org/en/latest/myql/ site_description: MYQL documentation site_author: Josue Kouka -use_directory_urls: True google_analytics: ['UA-32441224-4', 'myql.readthedocs.org'] pages: - Home : index.md From 47392df7254bd2990d50ca6f0f992d56dedcbeb5 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 9 Jul 2015 10:33:01 +0200 Subject: [PATCH 488/582] #1 : Updating a documentation --- docs/contrib.md | 3 ++- docs/table.md | 2 +- docs/weather.md | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/contrib.md b/docs/contrib.md index 9ed6273..d6301c7 100644 --- a/docs/contrib.md +++ b/docs/contrib.md @@ -4,7 +4,8 @@ It's easy to contribute to ***MYQL***. 1. Fork the repository 2. Develop your patches/fixes/features -3. Create a merge request +3. Test your changes ( > py2.6 ) +4. Submit a pull request That's all diff --git a/docs/table.md b/docs/table.md index 7f0fa9c..85be75a 100644 --- a/docs/table.md +++ b/docs/table.md @@ -169,7 +169,7 @@ As ***Binder***, ***BinderFunction*** is a subclass of ***BaseBinder***. They bo They say *"A picture is worth a thousand of words"* and I say *"A code snippet is worth ..."* . You got it (^_^). -Copy and paste the code snippet below in a *example.py* +Copy and paste the code snippet in the *example.py* below ```python from myql.contrib.table import BinderModel, InputKey, PagingPage, PagingUrl, InputValue, BinderFunction diff --git a/docs/weather.md b/docs/weather.md index 2c0dd81..b0fb3f3 100644 --- a/docs/weather.md +++ b/docs/weather.md @@ -1,4 +1,4 @@ -Weater +Weather ======= ### **Definition** From e53b26e7aac19098225533325bc0387980eeb1b2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Thu, 9 Jul 2015 22:47:48 +0200 Subject: [PATCH 489/582] fixing YQL import --- myql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/__init__.py b/myql/__init__.py index debf5a8..db4aa67 100755 --- a/myql/__init__.py +++ b/myql/__init__.py @@ -3,4 +3,4 @@ from myql import contrib from myql import errors from myql import utils -from myql.myql import MYQL +from myql.myql import YQL, MYQL From 26f81f8b8e7c5e1e32d8d23c4472211a9ef702f2 Mon Sep 17 00:00:00 2001 From: josuebrunel Date: Sat, 18 Jul 2015 18:49:48 +0200 Subject: [PATCH 490/582] #1: updating version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f957bcf..a292fdd 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ __author__ = 'Josue Kouka' __email__ = 'josuebrunel@gmail.com' -__version__ = "1.2.4" +__version__ = "1.2.5" #requirements.txt with open('requirements.txt') as f: From ef02818f5c99a57b0a587028e6e38de12386ecd3 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 27 Jul 2015 11:35:23 +0200 Subject: [PATCH 491/582] requests logs properly disabled --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 4a64547..156e5ca 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -11,7 +11,7 @@ logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n") logger = logging.getLogger('mYQL') -logging.getLogger('requests').setLevel(logging.WARNING) +logging.getLogger('requests').disabled = True # Disabling requests default logger class YQL(object): From ef78bc21df34d3752a16c0e285b3ab2fc11f678d Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Fri, 31 Jul 2015 11:43:53 +0200 Subject: [PATCH 492/582] version is taken from module --- myql/__init__.py | 2 ++ setup.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/myql/__init__.py b/myql/__init__.py index db4aa67..83e490c 100755 --- a/myql/__init__.py +++ b/myql/__init__.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +__version__ = '1.2.5' + from myql import contrib from myql import errors from myql import utils diff --git a/setup.py b/setup.py index f957bcf..42c437a 100755 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ import os +import myql from setuptools import setup, find_packages __author__ = 'Josue Kouka' __email__ = 'josuebrunel@gmail.com' -__version__ = "1.2.4" +__version__ = myql.__version__ #requirements.txt with open('requirements.txt') as f: From 92be4423b2ec932de0d777224e177a341583f837 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Fri, 31 Jul 2015 11:49:49 +0200 Subject: [PATCH 493/582] now using https for private url --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 4a64547..fdba9b5 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -24,7 +24,7 @@ class YQL(object): - community : set to to have access to community tables ''' public_url = 'https://query.yahooapis.com/v1/public/yql' - private_url = 'http://query.yahooapis.com/v1/yql' + private_url = 'https://query.yahooapis.com/v1/yql' community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): From 23c39c687d56a38c434ac563d71cf00bb5c9ceee Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Fri, 31 Jul 2015 19:29:39 +0200 Subject: [PATCH 494/582] yahoo_oauth version updated --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index a557ba1..2675f56 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -20,4 +20,4 @@ requests>=2.7.0 requests-oauthlib>=0.5.0 six>=1.9.0 tornado>=4.2 -yahoo-oauth>=0.1.4 +yahoo-oauth>=0.1.5 From 915d8b8414ef5ad8d685de81638689da2d8b96c0 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sat, 1 Aug 2015 12:35:03 +0200 Subject: [PATCH 495/582] #122: social te class added --- tests/__init__.py | 1 + tests/tests.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 281ee45..afef996 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,3 +5,4 @@ from tests.tests import TestStockScraper from tests.tests import TestTable from tests.tests import TestOAuth +from tests.tests import TestSocial diff --git a/tests/tests.py b/tests/tests.py index 69a4bc2..8a63a41 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -10,7 +10,7 @@ from yahoo_oauth import OAuth1 -from myql import MYQL +from myql import MYQL, YQL from myql.utils import pretty_xml, pretty_json, prettyfy from myql.contrib.table import Table @@ -303,6 +303,16 @@ def test_get_balancesheet(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) +class TestSocial(unittest.TestCase): + + def setUp(self,): + oauth = OAuth1(None, None, from_file='credentials.json') + self.yql = YQL(oauth=oauth) + + def test_get_contacts(self,): + data = self.yql.select('social.contacts').where(['guid','=','me']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) class TestTable(unittest.TestCase): From 435c5b08a3f80f27563ad01867d5db2221e42900 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sat, 1 Aug 2015 12:36:48 +0200 Subject: [PATCH 496/582] yahoo_oauth and requests logger disabled --- tests/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 69a4bc2..8b2993c 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -24,6 +24,10 @@ logging.getLogger('Test-mYQL') +logging.getLogger('mYQL').disabled = True +logging.getLogger('yahoo_oauth').disabled = True +logging.getLogger('requests').disabled = True + def json_write_data(json_data, filename): with open(filename, 'w') as fp: json.dump(json_data, fp, indent=4, sort_keys=True, ensure_ascii=False) From 7bf1c6b80e441bcad5be1540b21a7383a27318e0 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sat, 1 Aug 2015 13:52:24 +0200 Subject: [PATCH 497/582] updating coverage settings --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 88ee395..5417ab2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -24,5 +24,5 @@ exclude_lines = ignore_errors = True [html] -directory = html_report +directory = htmlcov From 7867b91a05d9866c96c6ebf04162ab1ef43a387f Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 3 Aug 2015 07:43:27 +0200 Subject: [PATCH 498/582] #141 : predefined word implemented --- myql/myql.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 50c3577..0703f7d 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -3,6 +3,7 @@ from __future__ import absolute_import +import re import logging import requests from myql import errors @@ -108,6 +109,8 @@ def clauseFormatter(self, cond): '''Formats conditions args is a list of ['column', 'operator', 'value'] ''' + + if cond[1].lower() == 'in': if not isinstance(cond[2], str) and 'select' not in cond[2][0].lower() : cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]]))) @@ -118,7 +121,11 @@ def clauseFormatter(self, cond): cond = " ".join(cond) else: - cond[2] = "'{0}'".format(cond[2]) + var = re.match('@(\w+)', cond[2]) + if var : + cond[2] = "{0}".format(var.group(1)) + else : + cond[2] = "'{0}'".format(cond[2]) cond = ''.join(cond) return cond From 6a09643118eb3df8f220aa53ceb892549cf188ff Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 3 Aug 2015 07:45:31 +0200 Subject: [PATCH 499/582] #122 #141 : tests improved --- tests/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 777a9be..7579ed7 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -24,9 +24,9 @@ logging.getLogger('Test-mYQL') -logging.getLogger('mYQL').disabled = True +#logging.getLogger('mYQL').disabled = True logging.getLogger('yahoo_oauth').disabled = True -logging.getLogger('requests').disabled = True +logging.getLogger('requests').setLevel(logging.CRITICAL) def json_write_data(json_data, filename): with open(filename, 'w') as fp: @@ -314,7 +314,7 @@ def setUp(self,): self.yql = YQL(oauth=oauth) def test_get_contacts(self,): - data = self.yql.select('social.contacts').where(['guid','=','me']) + data = self.yql.select('social.contacts').where(['guid','=','@me']) logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) From 117d300c5d3ece9265c18d500a3e12379ee55474 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 3 Aug 2015 10:21:17 +0200 Subject: [PATCH 500/582] #141 : fix issue when a none or is passed --- myql/myql.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 0703f7d..6c795a4 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -109,8 +109,6 @@ def clauseFormatter(self, cond): '''Formats conditions args is a list of ['column', 'operator', 'value'] ''' - - if cond[1].lower() == 'in': if not isinstance(cond[2], str) and 'select' not in cond[2][0].lower() : cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]]))) @@ -121,7 +119,10 @@ def clauseFormatter(self, cond): cond = " ".join(cond) else: - var = re.match('@(\w+)', cond[2]) + if isinstance(cond[2], (str, buffer)): + var = re.match('^@(\w+)$', cond[2]) + else: + var = None if var : cond[2] = "{0}".format(var.group(1)) else : From f08d4a8d7479ecec3ea9b660e1126e0ce8ec2bb5 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 3 Aug 2015 11:46:17 +0200 Subject: [PATCH 501/582] diagnostics added as default option --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 6c795a4..f74db54 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -28,7 +28,7 @@ class YQL(object): private_url = 'https://query.yahooapis.com/v1/yql' community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table - def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, oauth=None): + def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, diagnostics=False, oauth=None): self.community = community # True means access to community data self.format = format self._table = None From 65b0ffb84a8595ed8ab1a14013093634d8bc995e Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 3 Aug 2015 11:54:23 +0200 Subject: [PATCH 502/582] fix issue with diagnostics --- myql/myql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index f74db54..b50d888 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -34,7 +34,7 @@ def __init__(self, community=True, format='json', jsonCompact=False, crossProduc self._table = None self._query = None # used to build query when using methods such as , , ... self._payload = {} # Last payload self.diagnostics = diagnostics # Who knows, someone would like to turn it ON lol - self.limit = None + self._limit = None + self._offset = None self.crossProduct = crossProduct self.jsonCompact = jsonCompact self.debug = debug @@ -183,7 +184,7 @@ def desc(self, table=None): return response ##GET - def get(self, table=None, items=None, limit=''): + def get(self, table=None, items=None, limit=None, offset=None): '''Just a select which returns a response >>> yql.get("geo.countries', ['name', 'woeid'], 5") ''' @@ -192,7 +193,10 @@ def get(self, table=None, items=None, limit=''): items = ['*'] self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) if limit: - self._query += "limit {0}".format(limit) + self._query += "LIMIT {0} ".format(limit) + + if offset: + self._query += "OFFSET {0}".format(offset) if not self._table : raise errors.NoTableSelectedError('Please select a table') diff --git a/tests/__init__.py b/tests/__init__.py index 2295300..ce0d9bb 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,4 +7,4 @@ from tests.tests import TestOAuth from tests.tests import TestSocial from tests.tests import TestFilters -from tests.tests import TestPagingLimit +from tests.tests import TestPaging diff --git a/tests/tests.py b/tests/tests.py index 57e47fa..f84f421 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -159,24 +159,24 @@ def test_4_delete(self,): self.assertEqual(response.status_code, 200) -class TestPagingLimit(unittest.TestCase): +class TestPaging(unittest.TestCase): def setUp(self,): - self.yql = YQL() + self.yql = YQL(diagnostics=True, debug=True) def tearDown(self,): pass - def test_limit(self,): - data = self.yql.get('yql.table.list', limit=10) + def test_offset_get(self,): + data = self.yql.get('yql.table.list', limit=10, offset=3) logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) - def test_offset(self): - pass + def test_offset_select(self): + data = self.yql.select('geo.counties', limit=10, offset=3).where(['place', '=', 'CA']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) - def test_limit_offset(self,): - pass class TestFilters(unittest.TestCase): From f53555efa0914088f9fb944a4a2b34c1ecfd8cdb Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Fri, 14 Aug 2015 21:27:35 +0200 Subject: [PATCH 521/582] fix #143: support for OFFSET OK --- myql/myql.py | 9 +++++++-- tests/tests.py | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 1cdb8a2..bb67059 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -208,7 +208,7 @@ def get(self, table=None, items=None, limit=None, offset=None): ## SELECT - def select(self, table=None, items=None, limit=None): + def select(self, table=None, items=None, limit=None, offset=None): '''This method simulate a select on a table >>> yql.select('geo.countries', limit=5) >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) @@ -219,6 +219,7 @@ def select(self, table=None, items=None, limit=None): self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) self._limit = limit + self._offset = offset return self @@ -275,7 +276,11 @@ def where(self, *args): self._query += ' AND '.join(clause) if self._limit : - self._query += " LIMIT {0}".format(self._limit) + self._query += " LIMIT {0} ".format(self._limit) + + if self._offset : + self._query += " OFFSET {0} ".format(self._offset) + payload = self.payload_builder(self._query) response = self.execute_query(payload) diff --git a/tests/tests.py b/tests/tests.py index f84f421..727086a 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -166,6 +166,10 @@ def setUp(self,): def tearDown(self,): pass + def test_offset_raw_query(self,): + data = self.yql.raw_query("SELECT * FROM geo.counties WHERE place='CA' LIMIT 10 OFFSET 5 | sort(field='name')") + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) def test_offset_get(self,): data = self.yql.get('yql.table.list', limit=10, offset=3) From a0783d9ebd7bf11c1c7a608c6a8edec2a7be5726 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Fri, 14 Aug 2015 21:30:26 +0200 Subject: [PATCH 522/582] PEPPIFYING MYQL --- myql/myql.py | 6 +++--- tests/tests.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index bb67059..b293cc3 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -301,16 +301,16 @@ def __init__(self, *args, **kwargs): # ##################################################### - def getGUID(self, username): + def get_guid(self, username): '''Returns the guid of the username provided - >>> guid = self.getGUID('josue_brunel') + >>> guid = self.get_guid('josue_brunel') >>> guid ''' response = self.select('yahoo.identity').where(['yid', '=', username]) return response - def showTables(self, format='json'): + def show_tables(self, format='json'): '''Return list of all available tables''' query = 'SHOW TABLES' diff --git a/tests/tests.py b/tests/tests.py index 727086a..c7e2231 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -56,7 +56,7 @@ def test_desc(self,): def test_show_tables(self,): yql = MYQL(format='xml', community=False) - response = yql.showTables(format='xml') + response = yql.show_tables(format='xml') logging.debug(prettyfy(response, 'xml')) self.assertEqual(response.status_code, 200) @@ -252,7 +252,7 @@ def tearUp(self,): def test_get_guid(self,): oauth = OAuth1(None, None, from_file='credentials.json') yql = MYQL(format='json', oauth=oauth) - response = yql.getGUID('josue_brunel') + response = yql.get_guid('josue_brunel') logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) From 79aeaa49b54db8462e79434d2469714ac73c9381 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sat, 15 Aug 2015 08:11:11 +0200 Subject: [PATCH 523/582] #144: test remote filters start & count OK --- myql/myql.py | 6 ++++-- tests/__init__.py | 1 + tests/tests.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index b293cc3..40c37f3 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -208,15 +208,17 @@ def get(self, table=None, items=None, limit=None, offset=None): ## SELECT - def select(self, table=None, items=None, limit=None, offset=None): + def select(self, table=None, items=None, limit=None, offset=None, remote_filter=None): '''This method simulate a select on a table >>> yql.select('geo.countries', limit=5) >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) ''' self._table = table + if remote_filter: + table = "%s(%s)" %(table, ','.join(map(str, remote_filter))) if not items: items = ['*'] - self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) + self._query = "SELECT {1} FROM {0} ".format(table, ','.join(items)) self._limit = limit self._offset = offset diff --git a/tests/__init__.py b/tests/__init__.py index ce0d9bb..098093e 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,4 +7,5 @@ from tests.tests import TestOAuth from tests.tests import TestSocial from tests.tests import TestFilters +from tests.tests import TestRemoteFilters from tests.tests import TestPaging diff --git a/tests/tests.py b/tests/tests.py index c7e2231..fff7058 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -241,6 +241,25 @@ def test_filter_not_matches(self,): self.assertEqual(data.status_code, 200) +class TestRemoteFilters(unittest.TestCase): + + def setUp(self,): + self.yql = YQL(diagnostics=True, debug=True) + + def test_remote_filter_get(self,): + pass + + def test_remote_filter_select_count(self,): + data = self.yql.select('geo.counties', remote_filter=(20,)).where(['place', '=', 'CA']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + def test_remote_filter_select_start_and_count(self,): + data = self.yql.select('geo.counties', remote_filter=(60,20)).where(['place', '=', 'CA']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + class TestOAuth(unittest.TestCase): def setUp(self,): From 768fe4a239c73d76a4cc05ecc75f310a36764dbf Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sat, 15 Aug 2015 08:21:19 +0200 Subject: [PATCH 524/582] fix #144: all remote filters OK :+1: --- myql/myql.py | 10 ++++++++-- tests/tests.py | 12 ++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 40c37f3..d0f8759 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -184,14 +184,18 @@ def desc(self, table=None): return response ##GET - def get(self, table=None, items=None, limit=None, offset=None): + def get(self, table=None, items=None, limit=None, offset=None, remote_filter=None): '''Just a select which returns a response >>> yql.get("geo.countries', ['name', 'woeid'], 5") ''' self._table = table + + if remote_filter: + table = "%s(%s)" %(table, ','.join(map(str, remote_filter))) + if not items: items = ['*'] - self._query = "SELECT {1} FROM {0} ".format(self._table, ','.join(items)) + self._query = "SELECT {1} FROM {0} ".format(table, ','.join(items)) if limit: self._query += "LIMIT {0} ".format(limit) @@ -214,8 +218,10 @@ def select(self, table=None, items=None, limit=None, offset=None, remote_filter= >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) ''' self._table = table + if remote_filter: table = "%s(%s)" %(table, ','.join(map(str, remote_filter))) + if not items: items = ['*'] self._query = "SELECT {1} FROM {0} ".format(table, ','.join(items)) diff --git a/tests/tests.py b/tests/tests.py index fff7058..3fe9b74 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -246,8 +246,16 @@ class TestRemoteFilters(unittest.TestCase): def setUp(self,): self.yql = YQL(diagnostics=True, debug=True) - def test_remote_filter_get(self,): - pass + def test_remote_filter_get_count(self,): + data = self.yql.get('geo.countries', remote_filter=(10,)) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + + def test_remote_filter_get_start_and_count(self,): + data = self.yql.get('geo.countries', remote_filter=(100,10)) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + def test_remote_filter_select_count(self,): data = self.yql.select('geo.counties', remote_filter=(20,)).where(['place', '=', 'CA']) From 0156cb1b86601c56879acee68ea9532e94991d25 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sat, 15 Aug 2015 11:30:03 +0200 Subject: [PATCH 525/582] removing redandunt code : now get class select --- myql/myql.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index d0f8759..394bbd9 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -5,6 +5,7 @@ import re import logging + import requests from myql import errors @@ -188,23 +189,8 @@ def get(self, table=None, items=None, limit=None, offset=None, remote_filter=Non '''Just a select which returns a response >>> yql.get("geo.countries', ['name', 'woeid'], 5") ''' - self._table = table - - if remote_filter: - table = "%s(%s)" %(table, ','.join(map(str, remote_filter))) + self = self.select(table, items, limit, offset, remote_filter) - if not items: - items = ['*'] - self._query = "SELECT {1} FROM {0} ".format(table, ','.join(items)) - if limit: - self._query += "LIMIT {0} ".format(limit) - - if offset: - self._query += "OFFSET {0}".format(offset) - - if not self._table : - raise errors.NoTableSelectedError('Please select a table') - payload = self.payload_builder(self._query) response = self.execute_query(payload) From 4c649046e798920c2553dcc6f743b21e39027dc6 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sat, 15 Aug 2015 11:36:28 +0200 Subject: [PATCH 526/582] just improving the code --- myql/myql.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 394bbd9..d1a388e 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -185,17 +185,16 @@ def desc(self, table=None): return response ##GET - def get(self, table=None, items=None, limit=None, offset=None, remote_filter=None): + def get(self, *args, **kwargs): '''Just a select which returns a response >>> yql.get("geo.countries', ['name', 'woeid'], 5") ''' - self = self.select(table, items, limit, offset, remote_filter) + self = self.select(*args, **kwargs) payload = self.payload_builder(self._query) response = self.execute_query(payload) return response - ## SELECT def select(self, table=None, items=None, limit=None, offset=None, remote_filter=None): From 7fb012bbbedfa3ca3e770e1955641ed8422d9e37 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sat, 15 Aug 2015 18:56:17 +0200 Subject: [PATCH 527/582] #140: Post query filters tests added --- tests/tests.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 3fe9b74..09f6c4a 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -241,6 +241,32 @@ def test_filter_not_matches(self,): self.assertEqual(data.status_code, 200) +class TestPostQueryFilters(unittest.TestCase): + + def setUp(self,): + self.yql = YQL(diagnostics=True, debug=True) + + def tearDown(self,): + pass + + def test_post_filter_reverse(self,): + pass + + def test_post_filter_tail(self,): + pass + + def test_post_filter_truncate(self,): + pass + + def test_post_filter_sanitize(self,): + pass + + def test_post_filter_sort(self,): + pass + + def test_post_filter_unique(self,): + pass + class TestRemoteFilters(unittest.TestCase): def setUp(self,): From 327fc9eb626cb1c90355a341a517ff3824894d2a Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sun, 16 Aug 2015 14:17:29 +0200 Subject: [PATCH 528/582] Pythonifying the YQL Class --- myql/myql.py | 35 +++++++++++++++++++++++++---------- tests/__init__.py | 1 + tests/tests.py | 19 +++++++++++-------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index d1a388e..9a51cdf 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -25,9 +25,11 @@ class YQL(object): - diagnostics : set to to see diagnostics on queries - community : set to to have access to community tables ''' - public_url = 'https://query.yahooapis.com/v1/public/yql' - private_url = 'https://query.yahooapis.com/v1/yql' - community_data = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table + PUBLIC_URL = 'https://query.yahooapis.com/v1/public/yql' + PRIVATE_URL = 'https://query.yahooapis.com/v1/yql' + COMMUNITY_DATA = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table + + FUNC_FILTERS = ['sort', 'tail', 'truncate', 'reverse', 'unique', 'sanitize'] def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, diagnostics=False, oauth=None): self.community = community # True means access to community data @@ -53,7 +55,7 @@ def __repr__(self): def payload_builder(self, query, format=None): '''Build the payload''' if self.community : - query = self.community_data + query # access to community data tables + query = self.COMMUNITY_DATA + query # access to community data tables if vars(self).get('yql_table_url') : # Attribute only defined when MYQL.use has been called before query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query @@ -100,14 +102,14 @@ def execute_query(self, payload): if vars(self).get('oauth'): if not self.oauth.token_is_valid(): # Refresh token if token has expired self.oauth.refresh_token() - response = self.oauth.session.get(self.private_url, params= payload, header_auth=True) + response = self.oauth.session.get(self.PRIVATE_URL, params= payload, header_auth=True) else: - response = requests.get(self.public_url, params= payload) + response = requests.get(self.PUBLIC_URL, params= payload) self._response = response # Saving last response object. return response - def clause_formatter(self, cond): + def _clause_formatter(self, cond): '''Formats conditions args is a list of ['field', 'operator', 'value'] ''' @@ -138,7 +140,7 @@ def clause_formatter(self, cond): return cond - def buildResponse(self, response): + def response_builder(self, response): '''Try to return a pretty formatted response object ''' try: @@ -154,6 +156,16 @@ def buildResponse(self, response): return response + def __func_filters(self, filters): + '''Build post query filters + ''' + + for f in filters : + if isinstance(f, str) and f == 'reverse': + f = 'reverse()' + elif isinstance(f, dict) and f in YQL.FUNC_FILTERS: + pass + return ###################################################### # @@ -197,7 +209,7 @@ def get(self, *args, **kwargs): return response ## SELECT - def select(self, table=None, items=None, limit=None, offset=None, remote_filter=None): + def select(self, table=None, items=None, limit=None, offset=None, remote_filter=None, func_filters=None): '''This method simulate a select on a table >>> yql.select('geo.countries', limit=5) >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) @@ -207,6 +219,9 @@ def select(self, table=None, items=None, limit=None, offset=None, remote_filter= if remote_filter: table = "%s(%s)" %(table, ','.join(map(str, remote_filter))) + if func_filters: + pass + if not items: items = ['*'] self._query = "SELECT {1} FROM {0} ".format(table, ','.join(items)) @@ -263,7 +278,7 @@ def where(self, *args): self._query += ' WHERE ' for x in args: if x: - x = self.clause_formatter(x) + x = self._clause_formatter(x) clause.append(x) self._query += ' AND '.join(clause) diff --git a/tests/__init__.py b/tests/__init__.py index 098093e..283872c 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,4 +8,5 @@ from tests.tests import TestSocial from tests.tests import TestFilters from tests.tests import TestRemoteFilters +from tests.tests import TestFuncFilters from tests.tests import TestPaging diff --git a/tests/tests.py b/tests/tests.py index 09f6c4a..6d5f772 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -241,7 +241,7 @@ def test_filter_not_matches(self,): self.assertEqual(data.status_code, 200) -class TestPostQueryFilters(unittest.TestCase): +class TestFuncFilters(unittest.TestCase): def setUp(self,): self.yql = YQL(diagnostics=True, debug=True) @@ -249,22 +249,25 @@ def setUp(self,): def tearDown(self,): pass - def test_post_filter_reverse(self,): - pass + def test_func_filter_reverse(self,): + func_filters = 'reverse' + data = self.yql.select('geo.states', func_filters=func_filters).where(['place', '=', 'Congo']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) - def test_post_filter_tail(self,): + def test_func_filter_tail(self,): pass - def test_post_filter_truncate(self,): + def test_func_filter_truncate(self,): pass - def test_post_filter_sanitize(self,): + def test_func_filter_sanitize(self,): pass - def test_post_filter_sort(self,): + def test_func_filter_sort(self,): pass - def test_post_filter_unique(self,): + def test_func_filter_unique(self,): pass class TestRemoteFilters(unittest.TestCase): From 91b451526a197c7bb97188f9970cd2e1f60c3891 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sun, 16 Aug 2015 14:50:02 +0200 Subject: [PATCH 529/582] improbing the api --- myql/myql.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 9a51cdf..6fba0ea 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -52,7 +52,7 @@ def __repr__(self): ''' return ": {0} - : {1} ".format(self.community, self.format) - def payload_builder(self, query, format=None): + def _payload_builder(self, query, format=None): '''Build the payload''' if self.community : query = self.COMMUNITY_DATA + query # access to community data tables @@ -90,7 +90,7 @@ def raw_query(self, query, format=None, pretty=False): else: format = self.format - payload = self.payload_builder(query, format=format) + payload = self._payload_builder(query, format=format) response = self.execute_query(payload) if pretty: response = self.buildResponse(response) @@ -156,7 +156,7 @@ def response_builder(self, response): return response - def __func_filters(self, filters): + def __func_filters__(self, filters): '''Build post query filters ''' @@ -203,7 +203,7 @@ def get(self, *args, **kwargs): ''' self = self.select(*args, **kwargs) - payload = self.payload_builder(self._query) + payload = self._payload_builder(self._query) response = self.execute_query(payload) return response @@ -238,7 +238,7 @@ def insert(self, table,items, values): """ values = ["'{0}'".format(e) for e in values] self._query = "INSERT INTO {0} ({1}) VALUES ({2})".format(table,','.join(items),','.join(values)) - payload = self.payload_builder(self._query) + payload = self._payload_builder(self._query) response = self.execute_query(payload) return response @@ -290,7 +290,7 @@ def where(self, *args): self._query += " OFFSET {0} ".format(self._offset) - payload = self.payload_builder(self._query) + payload = self._payload_builder(self._query) response = self.execute_query(payload) return response @@ -322,7 +322,7 @@ def show_tables(self, format='json'): '''Return list of all available tables''' query = 'SHOW TABLES' - payload = self.payload_builder(query, format) + payload = self._payload_builder(query, format) response = self.execute_query(payload) From 7e563c3ec16fbf0fe6d26d5e5199278f571a709d Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sun, 16 Aug 2015 15:43:45 +0200 Subject: [PATCH 530/582] #140 : post query func filters reverse() OK :wink: --- myql/myql.py | 21 +++++++++++++-------- tests/tests.py | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 6fba0ea..286ea37 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -60,6 +60,9 @@ def _payload_builder(self, query, format=None): if vars(self).get('yql_table_url') : # Attribute only defined when MYQL.use has been called before query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query + if vars(self).get('_func'): # if post query function filters + query = '| '.join((query, self._func)) + self._query = query logger.info("QUERY = %s" %(query,)) @@ -156,16 +159,18 @@ def response_builder(self, response): return response - def __func_filters__(self, filters): + def _func_filters(self, filters): '''Build post query filters ''' + if not isinstance(filters, list): + raise TypeError('func_filters must be a List') - for f in filters : - if isinstance(f, str) and f == 'reverse': - f = 'reverse()' + for i, func in enumerate(filters) : + if isinstance(func, str) and func == 'reverse': + filters[i] = 'reverse()' elif isinstance(f, dict) and f in YQL.FUNC_FILTERS: pass - return + return '| '.join(filters) ###################################################### # @@ -219,13 +224,13 @@ def select(self, table=None, items=None, limit=None, offset=None, remote_filter= if remote_filter: table = "%s(%s)" %(table, ','.join(map(str, remote_filter))) - if func_filters: - pass - if not items: items = ['*'] self._query = "SELECT {1} FROM {0} ".format(table, ','.join(items)) + if func_filters: + self._func = self._func_filters(func_filters) + self._limit = limit self._offset = offset diff --git a/tests/tests.py b/tests/tests.py index 6d5f772..e8dfb60 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -250,7 +250,7 @@ def tearDown(self,): pass def test_func_filter_reverse(self,): - func_filters = 'reverse' + func_filters = ['reverse'] data = self.yql.select('geo.states', func_filters=func_filters).where(['place', '=', 'Congo']) logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) From 0d5d378c68278264f197bb6ebca7107172a8b8fc Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sun, 16 Aug 2015 16:03:16 +0200 Subject: [PATCH 531/582] #140 : post query func filters tail OK :+1: --- myql/myql.py | 8 ++++++-- tests/tests.py | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 286ea37..eb0cc96 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -163,13 +163,17 @@ def _func_filters(self, filters): '''Build post query filters ''' if not isinstance(filters, list): - raise TypeError('func_filters must be a List') + raise TypeError('func_filters must be a ') for i, func in enumerate(filters) : if isinstance(func, str) and func == 'reverse': filters[i] = 'reverse()' - elif isinstance(f, dict) and f in YQL.FUNC_FILTERS: + elif isinstance(func, tuple) and func[0] in YQL.FUNC_FILTERS: + filters[i] = '{:s}(count={:d})'.format(*func) + elif isinstance(func, dict) and func in YQL.FUNC_FILTERS: pass + else: + raise TypeError('{0} is neither a , a or a '.format(func)) return '| '.join(filters) ###################################################### diff --git a/tests/tests.py b/tests/tests.py index e8dfb60..c27f160 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -256,7 +256,10 @@ def test_func_filter_reverse(self,): self.assertEqual(data.status_code, 200) def test_func_filter_tail(self,): - pass + func_filters = [('tail', 2)] + data = self.yql.select('geo.states', func_filters=func_filters).where(['place', '=', 'Congo']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) def test_func_filter_truncate(self,): pass From ef0b37a7c697fd0a0662cfb9a1fb23df96e8679a Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sun, 16 Aug 2015 16:04:31 +0200 Subject: [PATCH 532/582] #140 : post query func filters truncate OK :+1: --- tests/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index c27f160..92a01e2 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -262,7 +262,11 @@ def test_func_filter_tail(self,): self.assertEqual(data.status_code, 200) def test_func_filter_truncate(self,): - pass + func_filters = [('truncate', 2)] + data = self.yql.select('geo.states', func_filters=func_filters).where(['place', '=', 'Congo']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + def test_func_filter_sanitize(self,): pass From 341cf4fc0e20fb8dbe63ee503f9a2a6dd957d65d Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Sun, 16 Aug 2015 17:27:30 +0200 Subject: [PATCH 533/582] fix #140 : post query func filters sort OK :+1: --- myql/myql.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index eb0cc96..faa8ddb 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -170,8 +170,19 @@ def _func_filters(self, filters): filters[i] = 'reverse()' elif isinstance(func, tuple) and func[0] in YQL.FUNC_FILTERS: filters[i] = '{:s}(count={:d})'.format(*func) - elif isinstance(func, dict) and func in YQL.FUNC_FILTERS: - pass + elif isinstance(func, dict) : + func_stmt = '' + import pdb + #pdb.set_trace() + try: + func_name = func.keys()[0] + except: #Py3 + func_name = list(func)[0] + + value = [ "{0}='{1}'".format(v[0], v[1]) for v in func[func_name] ] + func_stmt = ','.join(value) + func_stmt = '{0}({1})'.format(func_name, func_stmt) + filters[i] = func_stmt else: raise TypeError('{0} is neither a , a or a '.format(func)) return '| '.join(filters) From 5c8cab966174a347d2c5e9f59dba83104325ac4b Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 17 Aug 2015 14:15:25 +0200 Subject: [PATCH 534/582] control of value in IN filter added --- myql/myql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index faa8ddb..af6792a 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -96,7 +96,7 @@ def raw_query(self, query, format=None, pretty=False): payload = self._payload_builder(query, format=format) response = self.execute_query(payload) if pretty: - response = self.buildResponse(response) + response = self.response_builder(response) return response @@ -116,12 +116,15 @@ def _clause_formatter(self, cond): '''Formats conditions args is a list of ['field', 'operator', 'value'] ''' - + if len(cond) == 2 : cond = ' '.join(cond) return cond if 'in' in cond[1].lower() : + if not isinstance(cond[2], tuple): + raise TypeError('("{0}") must be of type '.format(cond[2])) + if not isinstance(cond[2], str) and 'select' not in cond[2][0].lower() : cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]]))) elif not isinstance(cond[2], str) and 'select' in cond[2][0].lower() : @@ -172,8 +175,6 @@ def _func_filters(self, filters): filters[i] = '{:s}(count={:d})'.format(*func) elif isinstance(func, dict) : func_stmt = '' - import pdb - #pdb.set_trace() try: func_name = func.keys()[0] except: #Py3 @@ -309,7 +310,6 @@ def where(self, *args): if self._offset : self._query += " OFFSET {0} ".format(self._offset) - payload = self._payload_builder(self._query) response = self.execute_query(payload) From cd5467ae55b8ec29858a59748339e8362900a602 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 17 Aug 2015 22:13:39 +0200 Subject: [PATCH 535/582] improving test coverage --- tests/tests.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 92a01e2..c324c7a 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -267,15 +267,38 @@ def test_func_filter_truncate(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) - def test_func_filter_sanitize(self,): + #func_filters = [('sanitize', '')] + #data = self.yql.select('geo.states', func_filters=func_filters).where(['place', '=', 'Congo']) + #logging.debug(pretty_json(data.content)) + #self.assertEqual(data.status_code, 200) pass def test_func_filter_sort(self,): - pass + func_filters = [ + {'sort': [ + ('field','name'), + ('descending','true') + ]}, + ('tail', 10), + #('reverse') + ] + data = self.yql.select('geo.counties', func_filters=func_filters).where(['place', '=', 'CA']) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) def test_func_filter_unique(self,): - pass + func_filters = [ + {'unique': [ + ('field','content'), + ('hideRepeatCount','false') + ]}, + ('truncate', 5) + ] + data = self.yql.get('yql.table.list', func_filters=func_filters) + logging.debug(pretty_json(data.content)) + self.assertEqual(data.status_code, 200) + class TestRemoteFilters(unittest.TestCase): From 24b3623e668dd548d1574b1c47de7ad195b20b0f Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 08:57:22 +0200 Subject: [PATCH 536/582] assert raise execption added for IN without tuple --- tests/tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index c324c7a..ed924f4 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -62,7 +62,7 @@ def test_show_tables(self,): def test_use(self): self.yql.use('http://www.josuebrunel.org/users.xml',name='users') - response = self.yql.raw_query('select * from users') + response = self.yql.raw_query('select * from users', format='xml') self.yql.yql_table_url = None try: logging.debug(pretty_json(response.content)) @@ -112,6 +112,12 @@ def test_select_in_2(self,): logging.error(e) self.assertEqual(response.status_code, 200) + def test_select_in_2_raise(self,): + + with self.assertRaises(TypeError): + response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','IN',('select woeid from geo.places(1) where text="Paris"')]) + + def test_1_insert(self,): response = self.yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) try: From 5191b550edf778753deae5156f6b1d6ea334d5c2 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 10:30:45 +0200 Subject: [PATCH 537/582] fix issue when List are passed to where with IN --- myql/myql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index af6792a..fa8d757 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -122,7 +122,7 @@ def _clause_formatter(self, cond): return cond if 'in' in cond[1].lower() : - if not isinstance(cond[2], tuple): + if not isinstance(cond[2], (tuple, list)): raise TypeError('("{0}") must be of type '.format(cond[2])) if not isinstance(cond[2], str) and 'select' not in cond[2][0].lower() : @@ -165,8 +165,8 @@ def response_builder(self, response): def _func_filters(self, filters): '''Build post query filters ''' - if not isinstance(filters, list): - raise TypeError('func_filters must be a ') + if not isinstance(filters, (list,tuple)): + raise TypeError('func_filters must be a or ') for i, func in enumerate(filters) : if isinstance(func, str) and func == 'reverse': From 4eaf358c48ea69fca195c9b08172d8b11b99a89f Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 10:31:31 +0200 Subject: [PATCH 538/582] test raise exception renamed --- tests/tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index ed924f4..564ad0c 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -112,12 +112,11 @@ def test_select_in_2(self,): logging.error(e) self.assertEqual(response.status_code, 200) - def test_select_in_2_raise(self,): + def test_raise_exception_select_where_in(self,): with self.assertRaises(TypeError): response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','IN',('select woeid from geo.places(1) where text="Paris"')]) - def test_1_insert(self,): response = self.yql.insert('yql.storage.admin',('value',),('http://josuebrunel.org',)) try: @@ -305,6 +304,11 @@ def test_func_filter_unique(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) + def test_raise_exception_func_filter(self): + + with self.assertRaises(TypeError): + pass + class TestRemoteFilters(unittest.TestCase): From b8e3347069706c71b1509a3beb5b7557ad0ce786 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 11:12:01 +0200 Subject: [PATCH 539/582] removing dummy try/except --- tests/tests.py | 64 +++++++++++++------------------------------------- 1 file changed, 16 insertions(+), 48 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 564ad0c..a005531 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -64,52 +64,34 @@ def test_use(self): self.yql.use('http://www.josuebrunel.org/users.xml',name='users') response = self.yql.raw_query('select * from users', format='xml') self.yql.yql_table_url = None - try: - logging.debug(pretty_json(response.content)) - except (Exception,) as e: - logging.error(e) + logging.debug(pretty_xml(response.content)) self.assertEqual(response.status_code, 200) def test_raw_query(self,): response = self.yql.raw_query('select name, woeid from geo.states where place="Congo"') - try: - logging.debug(pretty_json(response.content)) - except (Exception,) as e: - logging.error(e) + logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) def test_get(self,): self.yql.format = 'xml' response = self.yql.get('geo.countries', ['name', 'woeid'], 1) self.yql.format = 'json' - try: - logging.debug(pretty_xml(response.content)) - except (Exception,) as e: - logging.error(e) + logging.debug(pretty_xml(response.content)) self.assertEqual(response.status_code, 200) def test_select(self,): response = self.yql.select('geo.countries', ['name', 'code', 'woeid']).where(['name', '=', 'Canada']) - try: - logging.debug(pretty_json(response.content)) - except (Exception,) as e: - logging.error(e) + logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) def test_select_in(self,): response = self.yql.select('yahoo.finance.quotes').where(['symbol','in',("YHOO","AAPL","GOOG")]) - try: - logging.debug(pretty_json(response.content)) - except (Exception,) as e: - logging.error(e) + logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) def test_select_in_2(self,): response = self.yql.select('weather.forecast',['units','atmosphere']).where(['woeid','IN',('select woeid from geo.places(1) where text="Paris"',)]) - try: - logging.debug(pretty_json(response.content)) - except (Exception,) as e: - logging.error(e) + logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) def test_raise_exception_select_where_in(self,): @@ -133,44 +115,30 @@ def test_1_insert(self,): def test_2_check_insert(self,): json_data = json_get_data('yql_storage.json') response = self.yql.select('yql.storage').where(['name','=',json_data['select']]) - try: - logging.debug(pretty_json(response.content)) - except (Exception,) as e: - logging.error(response.content) - logging.error(e) - + logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) def test_3_update(self,): json_data = json_get_data('yql_storage.json') response = self.yql.update('yql.storage',('value',),('https://josuebrunel.org',)).where(['name','=',json_data['update']]) - try: - logging.debug(pretty_json(response.content)) - except (Exception,) as e: - logging.error(response.content) - logging.error(e) - + logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) def test_4_delete(self,): json_data = json_get_data('yql_storage.json') response = self.yql.delete('yql.storage').where(['name','=',json_data['update']]) - try: - logging.debug(pretty_json(response.content)) - except (Exception,) as e: - logging.error(response.content) - logging.error(e) - + logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) class TestPaging(unittest.TestCase): def setUp(self,): - self.yql = YQL(diagnostics=True, debug=True) + self.yql = YQL() def tearDown(self,): pass + def test_offset_raw_query(self,): data = self.yql.raw_query("SELECT * FROM geo.counties WHERE place='CA' LIMIT 10 OFFSET 5 | sort(field='name')") logging.debug(pretty_json(data.content)) @@ -249,7 +217,7 @@ def test_filter_not_matches(self,): class TestFuncFilters(unittest.TestCase): def setUp(self,): - self.yql = YQL(diagnostics=True, debug=True) + self.yql = YQL() def tearDown(self,): pass @@ -305,15 +273,15 @@ def test_func_filter_unique(self,): self.assertEqual(data.status_code, 200) def test_raise_exception_func_filter(self): - + func_filters = 'unique' with self.assertRaises(TypeError): - pass + data = self.yql.get('yql.table.list', func_filters=func_filters) class TestRemoteFilters(unittest.TestCase): def setUp(self,): - self.yql = YQL(diagnostics=True, debug=True) + self.yql = YQL(diagnostics=True, ) def test_remote_filter_get_count(self,): data = self.yql.get('geo.countries', remote_filter=(10,)) @@ -487,7 +455,7 @@ class TestSocial(unittest.TestCase): def setUp(self,): self.oauth = OAuth1(None, None, from_file='credentials.json') - self.yql = YQL(debug=True, diagnostics=True, oauth=self.oauth) + self.yql = YQL(oauth=self.oauth) def test_get_contacts(self,): data = self.yql.select('social.contacts').where(['guid','=', '@me']) From 63bd3aab1b6ad2c7a614eb6e10b9222c13d48149 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 11:18:04 +0200 Subject: [PATCH 540/582] test if remote_filter not a tuple OK --- myql/myql.py | 3 +++ tests/tests.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/myql/myql.py b/myql/myql.py index fa8d757..6a26bf2 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -238,6 +238,9 @@ def select(self, table=None, items=None, limit=None, offset=None, remote_filter= self._table = table if remote_filter: + if not isinstance(remote_filter, tuple): + raise TypeError("{0} must be of type ".format(remote_filter)) + table = "%s(%s)" %(table, ','.join(map(str, remote_filter))) if not items: diff --git a/tests/tests.py b/tests/tests.py index a005531..a61ae9e 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -304,6 +304,11 @@ def test_remote_filter_select_start_and_count(self,): logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) + def test_raise_exception_remote_filter_not_tuple(self,): + + with self.assertRaises(TypeError): + data = self.yql.get('geo.countries', remote_filter=10) + class TestOAuth(unittest.TestCase): From df2eb53ca55f39fc0e9e2c72259dd4ba0ad57e95 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 14:10:25 +0200 Subject: [PATCH 541/582] removing dummy if else --- myql/logger.py | 30 ++++++++++++++++++++++++++++++ myql/myql.py | 6 ++---- tests/tests.py | 2 +- 3 files changed, 33 insertions(+), 5 deletions(-) create mode 100755 myql/logger.py diff --git a/myql/logger.py b/myql/logger.py new file mode 100755 index 0000000..23897d4 --- /dev/null +++ b/myql/logger.py @@ -0,0 +1,30 @@ +from __future__ import absolute_import, unicode_literals + +import logging +from logging import Logger, handlers + +class YQLLogger(logging.Logger): + """Yahoo Logger class + """ + + def __init__(self, name, level=logging.DEBUG,filename=None, log_file_size=5*1024*1024): + """ + - name : logger name + - filename : file containing logs + """ + super(YahooLogger, self).__init__(name) + self.name = name + self.level = level + + self.setLevel(self.level) + + formatter = logging.Formatter("[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s") + if filename: + file_handler = handlers.RotatingFileHandler(filename, 'w', log_file_size, 10) + file_handler.setFormatter(formatter) + self.addHandler(file_handler) + else: + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + self.addHandler(stream_handler) + diff --git a/myql/myql.py b/myql/myql.py index 6a26bf2..6d73238 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -123,12 +123,10 @@ def _clause_formatter(self, cond): if 'in' in cond[1].lower() : if not isinstance(cond[2], (tuple, list)): - raise TypeError('("{0}") must be of type '.format(cond[2])) + raise TypeError('("{0}") must be of type or '.format(cond[2])) - if not isinstance(cond[2], str) and 'select' not in cond[2][0].lower() : + if 'select' not in cond[2][0].lower() : cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]]))) - elif not isinstance(cond[2], str) and 'select' in cond[2][0].lower() : - cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]]))) else: cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]]))) diff --git a/tests/tests.py b/tests/tests.py index a61ae9e..6f2266d 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -433,7 +433,7 @@ def test_get_index_summary(self,): def test_get_industry_index(self,): data = self.stock.get_industry_index(112) - logging.debug(pretty_json(data.content)) + #logging.debug(pretty_json(data.content)) self.assertEqual(data.status_code, 200) def test_get_symbols(self,): From 7b3ec144de630a79273d45db7fc0b56036114b4e Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 14:22:52 +0200 Subject: [PATCH 542/582] table argument is now mandatory --- myql/myql.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 6d73238..b3f3c98 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -202,14 +202,11 @@ def use(self, url, name='mytable'): return {'table url': url, 'table name': name} ##DESC - def desc(self, table=None): + def desc(self, table): '''Returns table description >>> yql.desc('geo.countries') >>> ''' - if not table: - #query = "desc {0} ".format(self._table) - raise errors.NoTableSelectedError('No table selected') query = "desc {0}".format(table) response = self.raw_query(query) @@ -228,7 +225,7 @@ def get(self, *args, **kwargs): return response ## SELECT - def select(self, table=None, items=None, limit=None, offset=None, remote_filter=None, func_filters=None): + def select(self, table, items=None, limit=None, offset=None, remote_filter=None, func_filters=None): '''This method simulate a select on a table >>> yql.select('geo.countries', limit=5) >>> yql.select('social.profile', ['guid', 'givenName', 'gender']) From 52bad43d9220b6c3ba625c4b472516d20f3caff4 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 14:46:15 +0200 Subject: [PATCH 543/582] jsonCompact enabled by default --- myql/myql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index b3f3c98..6f4258a 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -31,7 +31,7 @@ class YQL(object): FUNC_FILTERS = ['sort', 'tail', 'truncate', 'reverse', 'unique', 'sanitize'] - def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, diagnostics=False, oauth=None): + def __init__(self, community=True, format='json', jsonCompact=True, crossProduct=None, debug=False, diagnostics=False, oauth=None): self.community = community # True means access to community data self.format = format self._table = None @@ -72,7 +72,7 @@ def _payload_builder(self, query, format=None): 'diagnostics': self.diagnostics, 'format': format if format else self.format, 'debug': self.debug, - 'jsonCompact': self.jsonCompact + 'jsonCompact': 'new' if self.jsonCompact else '' } if self.crossProduct: payload['crossProduct'] = self.crossProduct From 0eb857a39d80fe748297091840aa910647cabcac Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 15:05:16 +0200 Subject: [PATCH 544/582] fix #145: crossProduct now implemented --- myql/myql.py | 2 +- tests/tests.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/myql/myql.py b/myql/myql.py index 6f4258a..ca71d8c 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -75,7 +75,7 @@ def _payload_builder(self, query, format=None): 'jsonCompact': 'new' if self.jsonCompact else '' } if self.crossProduct: - payload['crossProduct'] = self.crossProduct + payload['crossProduct'] = 'optimized' self._payload = payload logger.info("PAYLOAD = %s " %(payload, )) diff --git a/tests/tests.py b/tests/tests.py index 6f2266d..4b191fd 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -130,6 +130,12 @@ def test_4_delete(self,): logging.debug(pretty_json(response.content)) self.assertEqual(response.status_code, 200) + def test_cross_product(self): + yql = YQL(format='xml', crossProduct=True) + response = yql.select('weather.forecast').where(['location', '=', '90210']) + logging.debug("{0} {1}".format(response.status_code, response.reason)) + self.assertEqual(response.status_code, 200) + class TestPaging(unittest.TestCase): From cb8e26b3c002bc07a762cbfcf8d5c7e09131efdf Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 15:19:45 +0200 Subject: [PATCH 545/582] improving tests coverage --- tests/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 4b191fd..52dbd38 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -11,6 +11,7 @@ from yahoo_oauth import OAuth1 from myql import MYQL, YQL +from myql.errors import NoTableSelectedError from myql.utils import pretty_xml, pretty_json, prettyfy from myql.contrib.table import Table @@ -136,6 +137,10 @@ def test_cross_product(self): logging.debug("{0} {1}".format(response.status_code, response.reason)) self.assertEqual(response.status_code, 200) + def test_raise_exception_no_table_selected(self): + with self.assertRaises(NoTableSelectedError): + response = self.yql.select(None).where([]) + class TestPaging(unittest.TestCase): From 378b1e77976a758fabe56fdff426b665b5dad28a Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 15:30:48 +0200 Subject: [PATCH 546/582] cleaning the exception house --- myql/errors.py | 2 -- myql/logger.py | 30 ------------------------------ 2 files changed, 32 deletions(-) delete mode 100755 myql/logger.py diff --git a/myql/errors.py b/myql/errors.py index 92258e5..c1032c4 100755 --- a/myql/errors.py +++ b/myql/errors.py @@ -2,8 +2,6 @@ class NoTableSelectedError(Exception): '''Error raised when no table has been selected ''' def __init__(self, msg=None): - if not msg: - msg = 'No table selected' self.msg = msg def __str__(self): diff --git a/myql/logger.py b/myql/logger.py deleted file mode 100755 index 23897d4..0000000 --- a/myql/logger.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import absolute_import, unicode_literals - -import logging -from logging import Logger, handlers - -class YQLLogger(logging.Logger): - """Yahoo Logger class - """ - - def __init__(self, name, level=logging.DEBUG,filename=None, log_file_size=5*1024*1024): - """ - - name : logger name - - filename : file containing logs - """ - super(YahooLogger, self).__init__(name) - self.name = name - self.level = level - - self.setLevel(self.level) - - formatter = logging.Formatter("[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s") - if filename: - file_handler = handlers.RotatingFileHandler(filename, 'w', log_file_size, 10) - file_handler.setFormatter(formatter) - self.addHandler(file_handler) - else: - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - self.addHandler(stream_handler) - From c938659766e653fa0eb5a2d92c055060fb19df7a Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 16:29:53 +0200 Subject: [PATCH 547/582] improving stuff. I don't know what anymore --- myql/myql.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index ca71d8c..06cb556 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -186,6 +186,13 @@ def _func_filters(self, filters): raise TypeError('{0} is neither a , a or a '.format(func)) return '| '.join(filters) + def _add_limit(self,): + return ''.join((self._query," LIMIT {0} ".format(self._limit))) if self._limit else self._query + + def _add_offset(self,): + return ''.join((self._query," OFFSET {0} ".format(self._offset))) if self._offset else self._query + + ###################################################### # # MAIN METHODS @@ -219,6 +226,9 @@ def get(self, *args, **kwargs): ''' self = self.select(*args, **kwargs) + self._query = self._add_limit() + self._query = self._add_offset() + payload = self._payload_builder(self._query) response = self.execute_query(payload) @@ -302,11 +312,8 @@ def where(self, *args): self._query += ' AND '.join(clause) - if self._limit : - self._query += " LIMIT {0} ".format(self._limit) - - if self._offset : - self._query += " OFFSET {0} ".format(self._offset) + self._query = self._add_limit() + self._query = self._add_offset() payload = self._payload_builder(self._query) response = self.execute_query(payload) From 348bb2568bab75ecfa0d437921e0a1faf546af37 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 17:23:08 +0200 Subject: [PATCH 548/582] improving test coverage --- tests/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 52dbd38..01692da 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -288,6 +288,11 @@ def test_raise_exception_func_filter(self): with self.assertRaises(TypeError): data = self.yql.get('yql.table.list', func_filters=func_filters) + def test_raise_exception_func_filter_invalid_type(self): + func_filters = [30] + with self.assertRaises(TypeError): + data = self.yql.get('yql.table.list', func_filters=func_filters) + class TestRemoteFilters(unittest.TestCase): From d09882a1f834083bc44bf6af81680b3e450fab39 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 18 Aug 2015 21:21:02 +0200 Subject: [PATCH 549/582] cleaning redundancy --- myql/myql.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 06cb556..efceccb 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -63,7 +63,11 @@ def _payload_builder(self, query, format=None): if vars(self).get('_func'): # if post query function filters query = '| '.join((query, self._func)) - self._query = query + self._query = query + + self._query = self._add_limit() + self._query = self._add_offset() + logger.info("QUERY = %s" %(query,)) payload = { @@ -226,9 +230,6 @@ def get(self, *args, **kwargs): ''' self = self.select(*args, **kwargs) - self._query = self._add_limit() - self._query = self._add_offset() - payload = self._payload_builder(self._query) response = self.execute_query(payload) @@ -312,9 +313,6 @@ def where(self, *args): self._query += ' AND '.join(clause) - self._query = self._add_limit() - self._query = self._add_offset() - payload = self._payload_builder(self._query) response = self.execute_query(payload) From 80d60331a040d47ccc572ba250ce179ebf868fc5 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Wed, 19 Aug 2015 22:27:31 +0200 Subject: [PATCH 550/582] trying to fix broken url --- mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 3f2f882..8f140d9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,8 @@ site_name: mYQL repo_url: https://github.com/josuebrunel/myql site_url: http://myql.readthedocs.org/en/latest/myql/ +use_directory_urls: false +strict: true site_description: MYQL documentation site_author: Josue Kouka google_analytics: ['UA-32441224-4', 'myql.readthedocs.org'] From 86be9977c7253ffba560e35c3d040ba250446dbb Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Thu, 20 Aug 2015 13:39:21 +0200 Subject: [PATCH 551/582] fix #141 : variable substitution OK --- myql/myql.py | 24 ++++++++++++++++++------ tests/tests.py | 17 +++++++++++++---- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index efceccb..177fb8e 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -78,6 +78,10 @@ def _payload_builder(self, query, format=None): 'debug': self.debug, 'jsonCompact': 'new' if self.jsonCompact else '' } + + if vars(self).get('_vars'): + payload.update(self._vars) + if self.crossProduct: payload['crossProduct'] = 'optimized' @@ -136,12 +140,13 @@ def _clause_formatter(self, cond): cond = " ".join(cond) else: - if isinstance(cond[2], str): - var = re.match('^@(\w+)$', cond[2]) - else: - var = None - if var : - cond[2] = "{0}".format(var.group(1)) + #if isinstance(cond[2], str): + # var = re.match('^@(\w+)$', cond[2]) + #else: + # var = None + #if var : + if isinstance(cond[2], str) and cond[2].startswith('@'): + cond[2] = "{0}".format(cond[2]) else : cond[2] = "'{0}'".format(cond[2]) cond = ' '.join(cond) @@ -212,6 +217,13 @@ def use(self, url, name='mytable'): self.yql_table_name = name return {'table url': url, 'table name': name} + ##SET + def set(self, myvars): + ''' + ''' + self._vars = myvars + return True + ##DESC def desc(self, table): '''Returns table description diff --git a/tests/tests.py b/tests/tests.py index 01692da..43a11d8 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -137,6 +137,15 @@ def test_cross_product(self): logging.debug("{0} {1}".format(response.status_code, response.reason)) self.assertEqual(response.status_code, 200) + def test_variable_substitution(self,): + yql = YQL() + var = {'home': 'Congo'} + yql.set(var) + + response = yql.select('geo.states', remote_filter=(5,)).where(['place', '=', '@home']) + logging.debug(pretty_json(response.content)) + self.assertEqual(response.status_code, 200) + def test_raise_exception_no_table_selected(self): with self.assertRaises(NoTableSelectedError): response = self.yql.select(None).where([]) @@ -478,10 +487,10 @@ def setUp(self,): self.oauth = OAuth1(None, None, from_file='credentials.json') self.yql = YQL(oauth=self.oauth) - def test_get_contacts(self,): - data = self.yql.select('social.contacts').where(['guid','=', '@me']) - logging.debug(pretty_json(data.content)) - self.assertEqual(data.status_code, 200) + #def test_get_contacts(self,): + # data = self.yql.select('social.contacts').where(['guid','=', '@me']) + # logging.debug(pretty_json(data.content)) + # self.assertEqual(data.status_code, 200) class TestTable(unittest.TestCase): From d46c0761abddeb8bc12fa2ea7745b08cd154d7b2 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Thu, 20 Aug 2015 14:42:22 +0200 Subject: [PATCH 552/582] finxing py3 issue with dict keys and --- myql/myql.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/myql/myql.py b/myql/myql.py index 177fb8e..3ae38b2 100755 --- a/myql/myql.py +++ b/myql/myql.py @@ -37,6 +37,7 @@ def __init__(self, community=True, format='json', jsonCompact=True, crossProduct self._table = None self._query = None # used to build query when using methods such as