Merge pull request #8 from olech2412/bug/multipleGerAndLessCat

Bug/multiple ger and less cat

essensGetter.py

@@ -1,8 +1,9 @@
 from requests_html import HTMLSession
 from bs4 import BeautifulSoup as bs
+from bs4.element import Tag, NavigableString
 import datetime
 import calendar
-from utils.formatting import remove_HTML, format_food_price
+from utils.formatting import remove_HTML, format_food_price, format_meals_from_list
 from utils.mail import send_Email
 import logging
 
@@ -17,62 +18,39 @@ def give_me_everything():
     return data
 
 
-# Fetches the prices from the website
-def fetch_prices():
-    data = soup.find_all(class_="meals__price")
-    return data
+# Fetches the category from the food
+def fetch_food_from_website():
+    html_class_meals = soup.find_all(class_="meals")[0].__getattribute__("contents")
+    food_categorys = soup.find_all(class_="title-prim")
 
+    list_of_categorys_index = list()
+    list_of_all_meals = list()
 
-# Fetches the category from the food
-def fetch_food_category():
-    data = soup.find_all(class_="title-prim")
-    return remove_HTML(data)
-
-
-# Fetches the names from the food
-def fetch_food():
-    data = soup.find_all(class_="meals__name")
-    list_of_food = list()
-    beilagen = list()
-    for x in range(len(data)):
-        try:
-            list_of_food.append(data[x].__getattribute__("contents")[0])
-            if data[x].findNext(class_="u-list-bare").__getattribute__("contents")[0] is not None:
-                try:
-                    for i in range(len(data[x].findNext(class_="u-list-bare").__getattribute__("contents"))):
-                        if data[x].findNext(class_="u-list-bare").__getattribute__("contents")[i] != "\n":
-                            beilagen.append(
-                                data[x].findNext(class_="u-list-bare").__getattribute__("contents")[i].get_text())
-                        if i + 1 == len(data[x].findNext(class_="u-list-bare").__getattribute__("contents")):
-                            if (len(beilagen) == 1):
-                                list_of_food.append(beilagen[0])
-                            elif (len(beilagen) > 1):
-                                beilagen = ", ".join(beilagen)
-                                list_of_food.append(beilagen)
-                            else:
-                                list_of_food.append("")
-                            beilagen = list()
-                except Exception as e:
-                    logging.warning("Warn: " + str(e) + " in " + str(data[x]))
-                    print("Warn: " + str(e) + " in " + str(data[x]))
-                    beilagen.append("")
-            else:
-                logging.warning("No beilagen found for meal: " + data[x].get_text())
-                print("No beilagen found for meal: " + data[x].get_text())
-                list_of_food.append("")
-        except AttributeError as attribute_error:
-            logging.warning("AttributeError: " + str(attribute_error) + " in " + str(data[x]))
-            print("AttributeError: " + str(attribute_error) + " in " + str(data[x]))
-            list_of_food.append("Keine Beilagen")
-
-    return list_of_food
+    for x in html_class_meals:
+        if isinstance(x,NavigableString):
+            html_class_meals.remove(x)
+
+    for x in food_categorys:
+        list_of_categorys_index.append(html_class_meals.index(x))
+    list_of_categorys_index.append(len(html_class_meals))
+
+    for x in food_categorys:
+        one_meal = list()
+        count = list_of_categorys_index[food_categorys.index(x)]
+        for y in html_class_meals[count:list_of_categorys_index[food_categorys.index(x) + 1]:1]:
+            one_meal.append(y)
+
+            if html_class_meals.index(y) == list_of_categorys_index[food_categorys.index(x) + 1] -1:
+                list_of_all_meals.append(one_meal)
+
+    return list_of_all_meals
 
 
 # don't do anything on weekends
 if calendar.day_name[datetime.date.today().weekday()] == "Saturday" \
         or calendar.day_name[datetime.date.today().weekday()] == "Sunday":
-    logging.info("Weekend -> no call on website and no other operations")
-    print("Weekend -> no call on website and no other operations")
+   logging.info("Weekend -> no call on website and no other operations")
+   print("Weekend -> no call on website and no other operations")
 else:
     url = "https://www.studentenwerk-leipzig.de/mensen-cafeterien/speiseplan?location=140"  # URL
     session = HTMLSession()  # Initialize HTML Session
@@ -85,6 +63,5 @@ def fetch_food():
 
         # give_me_everything() # Important to know which property's you can extract
         # convert the HTML List to usable data
-
-        foodprice = format_food_price(fetch_prices())  # call the function to convert the HTML Stuff to usable data
-        send_Email(food=fetch_food(), foodcategory=fetch_food_category(), foodprice=foodprice)
+        meals = format_meals_from_list(fetch_food_from_website())
+        send_Email(meals)

tests/Test_essensGetter.py

@@ -8,29 +8,9 @@ def test_give_me_everything(self):
         self.assertIsNot(essensGetter.give_me_everything(), None)
         self.assertIsNot(essensGetter.give_me_everything(), [])
 
-    def test_fetch_prices(self):
-        self.assertIsNot(essensGetter.fetch_prices(), None)
-        self.assertIsNot(essensGetter.fetch_prices(), [])
-        self.assertGreater(len(essensGetter.fetch_prices()), 0)
-
-        if len(essensGetter.fetch_prices()) == 1:
-            self.assertGreater(len(essensGetter.fetch_food()), 1)
-            self.assertLess(len(essensGetter.fetch_food()), 3)
-            self.assertEqual(len(essensGetter.fetch_food_category()), 1)
-        elif len(essensGetter.fetch_prices()) == 2:
-            self.assertGreater(len(essensGetter.fetch_food()), 2)
-            self.assertLess(len(essensGetter.fetch_food()), 5)
-            self.assertEqual(len(essensGetter.fetch_food_category()), 2)
-
-    def test_fetch_food_category(self):
-        self.assertIsNot(essensGetter.fetch_food_category(), None)
-        self.assertIsNot(essensGetter.fetch_food_category(), [])
-        self.assertGreater(len(essensGetter.fetch_food_category()), 0)
-
-    def test_fetch_food(self):
-        self.assertIsNot(essensGetter.fetch_food(), None)
-        self.assertIsNot(essensGetter.fetch_food(), [])
-        self.assertGreater(len(essensGetter.fetch_food()), 0)
+    def test_fetch_food_from_website(self):
+        self.assertIsNot(essensGetter.fetch_food_from_website(), None)
+        self.assertIsNot(essensGetter.fetch_food_from_website(), [])
 
 
 if __name__ == '__main__':

tests/Test_formatting.py

@@ -2,12 +2,41 @@
 import utils.formatting as formatting
 import essensGetter
 
+
 class Test_formatting(unittest.TestCase):
 
     def test_format_price(self):
-        self.assertFalse(str(formatting.format_food_price(essensGetter.fetch_food())).__contains__("<p"))
+        self.assertFalse(formatting.format_food_price(formatting.format_meals_from_list(essensGetter.fetch_food_from_website())[0]["price"]).__contains__("<p"))
+        self.assertFalse(formatting.format_food_price(formatting.format_meals_from_list(essensGetter.fetch_food_from_website())[0]["price"]).__contains__("</p"))
+        self.assertFalse(formatting.format_food_price(formatting.format_meals_from_list(essensGetter.fetch_food_from_website())[0]["price"]).__contains__("title-prim"))
 
     def test_format_string(self):
-        self.assertTrue(str(formatting.format_string(essensGetter.fetch_food())).isascii())
-        self.assertTrue(str(formatting.format_string(essensGetter.fetch_food_category())).isascii())
-        self.assertTrue(str(formatting.format_string(essensGetter.fetch_prices())).isascii())
+        for x in formatting.format_meals_from_list(essensGetter.fetch_food_from_website()):
+            for y in x["price"]:
+                self.assertTrue(formatting.format_string(str(y)).isascii())
+
+            self.assertTrue(formatting.format_string(str(x["category"])).isascii())
+            for y in x["food"]:
+                self.assertTrue(formatting.format_string(str(y)).isascii())
+            for y in x["beilagen"]:
+                self.assertTrue(formatting.format_string(str(y)).isascii())
+
+            if len(formatting.format_meals_from_list(essensGetter.fetch_food_from_website())) == 4:
+                self.assertRaises(KeyError, formatting.format_meals_from_list(essensGetter.fetch_food_from_website())[0]["additional_info"])
+            elif len(formatting.format_meals_from_list(essensGetter.fetch_food_from_website())) == 5:
+                self.assertTrue(str(formatting.format_string(formatting.format_meals_from_list(essensGetter.fetch_food_from_website())[0]["additional_info"])).isascii())
+
+    def test_format_meals_from_list(self):
+        for x in formatting.format_meals_from_list(essensGetter.fetch_food_from_website()):
+            self.assertTrue(x.__contains__("category"))
+            self.assertTrue(x.__contains__("food"))
+            self.assertTrue(x.__contains__("beilagen"))
+            self.assertTrue(x.__contains__("price"))
+            if len(x) == 5:
+                self.assertTrue(str(x.__contains__("additional_info")))
+            elif len(x) == 4:
+                self.assertFalse(x.__contains__("additional_info"))
+
+
+if __name__ == '__main__':
+    unittest.main()

utils/formatting.py

@@ -1,8 +1,10 @@
 import logging
+import traceback
 
+from bs4.element import NavigableString
 
 logging.basicConfig(filename='essensGetter.log', level=logging.INFO, filemode='w',
-                    format='%(asctime)s %(levelname)s - %(message)s', force=True, encoding='utf-8')
+                    format='%(asctime)s %(levelname)s %(lineno)d - %(message)s', force=True, encoding='utf-8')
 
 
 # Removes the HTML from the data
@@ -16,8 +18,11 @@ def remove_HTML(htmlContent):
             del (secondsplit[1])
             htmlContent[i] = secondsplit
     elif isinstance(htmlContent, str):
-        logging.error("A string is given -> no formatting ")
-        print("A string is given -> no formatting ")
+        first_split = htmlContent.split(">")
+        del (first_split[0])
+        secondsplit = first_split[0].split("</")
+        del (secondsplit[1])
+        htmlContent = secondsplit
     else:
         logging.error("Unknown datatype -> no formatting")
         print("Unknown datatype -> no formatting")
@@ -63,6 +68,7 @@ def format_string(nonAsciiStr):
     nonAsciiStr = nonAsciiStr.replace("Ô", "O")
     nonAsciiStr = nonAsciiStr.replace("Û", "U")
     nonAsciiStr = nonAsciiStr.replace("€", "Euro")
+    nonAsciiStr = nonAsciiStr.replace("\"", "")
     logging.debug("After formatting: " + str(nonAsciiStr) + "\n")
     return nonAsciiStr
 
@@ -76,4 +82,95 @@ def format_food_price(data):
         data[x] = str(data[x]).replace(" ", "")
     logging.info("Prices after formatting: " + str(data))
     print("Prices after formatting: " + str(data))
-    return data
+    return str(data)
+
+
+def format_meals_from_list(list_of_all_meals):
+    """Converts the list_of_all_meals into a dictionary with all special possibilities
+
+    Takes the fetched Data from essensGetter.py and convert it into dictionaries. Because multiple meals are possible
+    each dictionary will be added to a list.
+    :param list_of_all_meals:
+    :return: meals
+    """
+
+    meals = list()
+
+    for x in list_of_all_meals:
+        try:
+            if len(x) == 2:
+                meal = {"category": x[0].__getattribute__("contents")[0], "food": "", "beilagen": "", "price": ""}
+
+                html_food_content = x[1].__getattribute__("contents")
+                for y in html_food_content:
+                    if isinstance(y, NavigableString):
+                        html_food_content.remove(y)
+
+                food_list = list()
+                price_list = list()
+                for y in html_food_content:
+                    food_list.append(y.find_all(class_="meals__name")[0].__getattribute__("contents")[0])
+                    price_list.append(format_food_price(y.find_all(class_="meals__price")))
+                meal["food"] = food_list
+                meal["price"] = price_list
+
+                beilagen_list = list()
+                for y in html_food_content:
+                    try:
+                        for z in y.find_all(class_="u-list-bare")[0]:
+                            if isinstance(z, NavigableString) is False:
+                                beilagen_list.append(remove_HTML(str(z)))
+                    except Exception as e:
+                        logging.warning("No beilagen found")
+                        print("No beilagen found")
+                meal["beilagen"] = beilagen_list
+
+                meals.append(meal)
+        except Exception as e:
+            logging.error("Error in format_meals_from_list for a meal containing 2 Attr.: " + str(e))
+            print("Error in format_meals_from_list for a meal containing 2 Attr.: " + str(e))
+            continue
+
+        try:
+            if len(x) == 3:
+                meal = {"category": x[0].__getattribute__("contents")[0], "food": "", "beilagen": "", "price": "",
+                        "additional_info": ""}
+
+                try:
+                    meal["additional_info"] = x[1].__getattribute__("contents")[0]
+                except AttributeError as e:
+                    logging.error("No additional info available: " + str(e))
+                    print("No additional info available: " + str(e))
+                    meal["additional_info"] = ""
+
+                html_food_content = x[2].__getattribute__("contents")
+                for y in html_food_content:
+                    if isinstance(y, NavigableString):
+                        html_food_content.remove(y)
+
+                food_list = list()
+                price_list = list()
+                for y in html_food_content:
+                    food_list.append(y.find_all(class_="meals__name")[0].__getattribute__("contents")[0])
+                    price_list.append(format_food_price(y.find_all(class_="meals__price")))
+                meal["food"] = food_list
+                meal["price"] = price_list
+
+                beilagen_list = list()
+                for y in html_food_content:
+                    try:
+                        for z in y.find_all(class_="u-list-bare")[0]:
+                            if isinstance(z, NavigableString) is False:
+                                beilagen_list.append(z)
+                    except Exception as e:
+                        logging.warning("No beilagen available")
+                        print("No beilagen available")
+                meal["beilagen"] = beilagen_list
+
+                meals.append(meal)
+
+        except Exception as e:
+            logging.exception("Error in format_meals_from_list for a meal containing 3 Attr.: " + str(e))
+            print("Error in format_meals_from_list for a meal containing 3 Attr.: " + str(e))
+
+    return meals

utils/mail.py

@@ -7,26 +7,27 @@
                     format='%(asctime)s %(levelname)s - %(message)s', force=True, encoding='utf-8')
 
 
-def send_Email(food, foodcategory, foodprice):
-
-
-
-
-    # Gericht 1
-    food1 = format_string(foodcategory[0]) + " (" + format_string(foodprice[0]) + ")" + ": " \
-            + format_string(food[0]) + " [" + format_string(food[1]) + "]"
-    logging.info("Food1: " + food1)
-    print("Food1: " + food1)
-    # Gericht 2
-    food2 = format_string(foodcategory[1]) + " (" + format_string(foodprice[1]) + ")" \
-            + ": " + format_string(food[2]) + " [" + format_string(food[3]) + "]"
-    logging.info("Food2: " + food2)
-    print("Food2: " + food2)
+def send_Email(meals):
+    actual_meals = list()
+    for x in meals:
+        if len(x) == 5:
+            food_list_conv = list()
+
+            for y in x["food"]:
+                food_list_conv.append(format_string(y) + " - " + format_string(x["price"][x["food"].index(y)]) + "\n")
+
+            food = format_string(str(x["category"]) + " --" + str(x["additional_info"])) + "-- : \n" + "".join(
+                food_list_conv)
+            actual_meals.append(food)
+        elif len(x) == 4:
+            food = format_string(str(x["category"])) + ": " + "\n" + format_string(str(x["food"])) \
+                   + " [" + format_string(str(x["beilagen"])) + "]" + " - " + format_string(str(x["price"])) \
+                   + "\n"
+            actual_meals.append(food)
 
     current_day = str(datetime.date.strftime(datetime.date.today(), "%d.%m.%Y"))
 
-    # add all Emails an Names to the list
-    # TODO maybe use a dictionary instead of a list
+    # add all Emails and Names to the list
     try:
         with open("utils/receivers") as file:
             receivers = list()
@@ -46,7 +47,8 @@ def send_Email(food, foodcategory, foodprice):
             names.append(receivers[x])
 
     content = "Moin {}, \n \n" + "Schau dir an was es heute in der Kantine (Schoenauer Strasse) zu essen gibt: \n \n" \
-              + food1 + "\n" + food2 + "\n \n" + "Bis denne," + "\n" + "dein Food-Bot - " + current_day
+              + actual_meals[0] + "\n" + actual_meals[
+                  1] + "\n \n" + "Bis denne," + "\n" + "dein Food-Bot - " + current_day
 
     logging.info("Content: " + content)