Merge pull request 'making_image_jobs_optional' (#6) from making_image_jobs_optional into main

Reviewed-on: #6

Sonnendach.py

@@ -10,10 +10,18 @@ import os
 import qrcode
 from tkinter import filedialog
 import time
+import datetime
+from dateutil.relativedelta import relativedelta
 from selenium.webdriver.common.action_chains import ActionChains
+import json
+import requests
+import statistics
 
-DropDownLabels = ["Street: ", "Number: ", "Postal code: ", "City: ", "Sonnendach URL: ", "Suitability", "Image Filename Map: ", "Image Filename Production: ", "Image Filename qrcode: ", "PV Production 50", "PV Production 75", "PV Production 100", "Value Electricity production"]
+DropDownLabels = ["Street: ", "Number: ", "Postal code: ", "City: ", "coordinate X", "coordinate Y", "Sonnendach URL: ", "Suitability:", "Image Filename: ", "PV Production 100", "Value Electricity production", "Roof area"]
 file_split_char = ","
+minRoofSize = 2
+minSuitability = 3
+electricityPrice = 0.1
 OptionList = []
 exit = False
 stopThread = False
@@ -25,67 +33,272 @@ driver = webdriver.Chrome(service=service)
 driver.minimize_window()
 outputtext = "Welcome to application Sonnendach\n"
 columnIndexes = []
-
+storeToDolibarr = True
+dolibarrURL = ""
+dolibarrApiKey = ""
+dolibarrCountryID = "6" # 6 = Schweiz
+dolibarrCountryCode = "CH"
+dolibarrStateID = "403" # 403 = Kanton Zürich
 
 
 def search_adresses(adress_list, filename_adresslist, driver, mainText):
+    print("function to search Addresses called")
     global file_split_char
     global stopThread
     global columnIndexes
     global outputtext
-
+    global checkBoxCreateImages
+    global minRoofSize
+    global minSuitability
+    global electricityPrice
+    global storeToDolibarr
+    global dolibarrURL
+    global dolibarrApiKey
 
     #Create Direcotrys to save screenshots and qrcodes if they don't exist yet.
-    if(not os.path.exists("screenshots")):
+    image_folder_map = "screenshots_map/"
-        os.makedirs("screenshots", exist_ok=False)
+    image_folder_production = "screenshots_production/"
-    if (not os.path.exists("qrcodes")):
+    image_folder_qrcode = "qrcodes/"
-        os.makedirs("qrcodes", exist_ok=False)
+    if(not os.path.exists(image_folder_map[:-1])):
+        os.makedirs(image_folder_map[:-1], exist_ok=False)
+    if(not os.path.exists(image_folder_production[:-1])):
+        os.makedirs(image_folder_production[:-1], exist_ok=False)
+    if (not os.path.exists(image_folder_qrcode[:-1])):
+        os.makedirs(image_folder_qrcode[:-1], exist_ok=False)
 
+    startTime = datetime.datetime.now()
+    address_count = len(adress_list)
     #Search for each adress in the list and create screenshots and qrcodes
-    for i in range(len(adress_list)):
+    print("Starting Address Loop")
+    for i in range(address_count):
+        url = "not-found"
+        suitability = "not-found"
+        sumEnergyRoofs = "not-found"
+        value_electricity_production = "not-found"
+        sumRoofArea = "not-found"
+        featureId = "not-found"
+        response_all_roofs_decoded = "not-found"
+        found = True
         line = adress_list[i]
         adress = line.split(file_split_char)
-        if((line != adress_list[0]) & (adress[columnIndexes[4]] == "") & (len(adress[columnIndexes[0]]) > 0) & (len(adress[columnIndexes[1]]) > 0) & (len(adress[columnIndexes[2]]) > 0) & (len(adress[columnIndexes[3]]) > 0)):
+        if((line != adress_list[0]) and (adress[columnIndexes[6]] == "") and (((len(adress[columnIndexes[0]]) > 0) and (len(adress[columnIndexes[1]]) > 0) and (len(adress[columnIndexes[2]]) > 0) and (len(adress[columnIndexes[3]]) > 0)) or ((len(adress[columnIndexes[4]]) > 0) and (len(adress[columnIndexes[5]]) > 0)))):
-            print(adress)
+            address_string = adress[columnIndexes[0]] + " " + adress[columnIndexes[1]] + " " + adress[columnIndexes[2]] + " " + adress[columnIndexes[3]]
-            search_string = adress[columnIndexes[0]] + " " + adress[columnIndexes[1]] + " " + adress[columnIndexes[2]] + " " + adress[columnIndexes[3]]
+            print("searching for Address: " + address_string)
-            search_bar = driver.find_element(By.ID, "searchTypeahead1")
+            if(not ((len(adress[columnIndexes[4]]) > 0) and (len(adress[columnIndexes[5]]) > 0))):
-            search_bar.send_keys(Keys.CONTROL + "a")
+                # Get Coordinates for Address
-            search_bar.send_keys(Keys.DELETE)
+                try:
-            search_bar.send_keys(search_string)
+                    # Address to coordinates
-            driver.implicitly_wait(10)
+                    params = {
+                        "type": "locations",
+                        "searchText": address_string,
+                        "lang": "de"}
+                    locationsResponse = requests.get('https://api3.geo.admin.ch/rest/services/api/SearchServer',
+                                                     params=params)
+                    locations = json.loads(locationsResponse.content.decode())
+                    firstLocation = locations['results'][0]['attrs']
 
+                    # you need the coordinates from other services, once you have them
+                    coordinates = (float(firstLocation['y']), float(firstLocation['x']))  # example coordinates
+                except:
                     found = False
-            suggestions = driver.find_elements(By.XPATH, "//div[@class='tt-suggestion tt-selectable']")
+            else:
-            for suggestion in suggestions:
+                # Get Coordinates from File
-                if(suggestion.text == search_string):
+                coordinate_x = float(adress[columnIndexes[4]]) % 1000000
-                    found = True
+                coordinate_y = float(adress[columnIndexes[5]]) % 1000000
-                    suggestion.click()
+                coordinates = (coordinate_x, coordinate_y)
-                    driver.implicitly_wait(10)
-                    break
 
             if(found):
+                # Get Building ID from coordinates
+                try:
+                    coordinates_str = ','.join(map(str, coordinates))
+                    tolerance = 1
+                    # this is wrong, bbox should be computed using tolerance and pixel size, but there are no problems for the
+                    # purpose of this exercise
+                    bbox = (coordinates[0] - tolerance, coordinates[1] - tolerance, coordinates[0] + tolerance,
+                            coordinates[1] + tolerance)
+                    bbox_str = ','.join(map(str, bbox))
+                    params = {
+                        "geometryType": "esriGeometryPoint",
+                        "returnGeometry": "true",
+                        "layers": "all:ch.bfe.solarenergie-eignung-daecher",
+                        "geometry": coordinates_str,
+                        "mapExtent": bbox_str,
+                        "imageDisplay": "1391,1070,96",
+                        "tolerance": tolerance,
+                        "order": "distance",
+                        "lang": "de",
+                    }
+                    # retrieve one roof and the building_id associated
+                    response_one_roof = requests.get('https://api3.geo.admin.ch/rest/services/api/MapServer/identify',
+                                                     params=params)
+                    response_one_roof_decoded = json.loads(response_one_roof.content.decode())
+                    results = (response_one_roof_decoded['results'])
+                    # for result in results:
+                    #     print(result)
+                    building_id = (response_one_roof_decoded)['results'][0]['attributes']['building_id']
+                except:
+                    found = False
+
+            if(found):
+                try:
+                    # use the building_id of previous request to get all roofs associated with this building_id
+                    params = {
+                        'layer': 'ch.bfe.solarenergie-eignung-daecher',
+                        'searchField': 'building_id',
+                        'contains': 'false',
+                        'searchText': str(building_id),
+                        'returnGeometry': 'false'
+                    }
+                    response_all_roofs = requests.get('https://api3.geo.admin.ch/rest/services/api/MapServer/find',
+                                                      params=params)
+                    response_all_roofs_decoded = json.loads(response_all_roofs.content.decode())['results']
+                    if(len(response_all_roofs_decoded) > 0):
+                        sumEnergyRoofs = 0
+                        sumRoofArea = 0
+                        suitabilities = []
+                        bestRoofEnergy = 0
+                        bestRoof = response_all_roofs_decoded[0]
+                        for roof in response_all_roofs_decoded:
+                            energy = float(roof['attributes']['gstrahlung'])*0.2*0.8
+                            if(energy > bestRoofEnergy):
+                                bestRoofEnergy = energy
+                                bestRoof = roof
+                            if(float(roof['attributes']['flaeche']) > minRoofSize and int(roof['attributes']['klasse']) >= minSuitability):
+                                sumEnergyRoofs = sumEnergyRoofs + energy
+                                sumRoofArea = sumRoofArea + float(roof['attributes']['flaeche'])
+                                suitabilities.append(int(roof['attributes']['klasse']) * float(roof['attributes']['flaeche']))
+
+                        featureId = str(bestRoof['featureId'])
+                        url = "https://www.uvek-gis.admin.ch/BFE/sonnendach/index.html?featureId=" + featureId + "&lang=de"
+                        suitability = statistics.mean(suitabilities) / sumRoofArea
+                        value_electricity_production = sumEnergyRoofs * electricityPrice
+                    else:
+                        found = False
+                except:
+                    found = False
+
+            roof_is_suitable = True # TODO decide if roof is suitable
+
+            if(found and createImages.get() > 0 and roof_is_suitable):
+                image_filename = suitability + " - " + address_string + ".png"
+                create_images(address_string, driver, image_filename, image_folder_map, image_folder_production, image_folder_qrcode,
+                              url)
+
+            else:
+                image_filename = "not-created"
+
+            print("saving Address: " + address_string)
+            save_information_to_addresslist(adress, adress_list, columnIndexes, coordinates, file_split_char, i,
+                                            image_filename, sumEnergyRoofs, sumRoofArea, suitability, url,
+                                            value_electricity_production)
+
+            save_addresslist_in_file(adress_list, filename_adresslist)
+
+
+            if(found):
+                street = adress[columnIndexes[0]] + " " + adress[columnIndexes[1]]
+                plz = adress[columnIndexes[2]]
+                city = adress[columnIndexes[3]]
+                create_Partner_in_dolibarr(street, plz, city, coordinates, sumEnergyRoofs, sumRoofArea, suitability, featureId, url, response_all_roofs_decoded)
+
+            print("saved Address: " + address_string)
+            print("Address " + str(i) + " of " + str(address_count))
+            timediff = relativedelta(datetime.datetime.now(), startTime)
+            print("Process is running since %d years %d months %d days %d hours %d minutes %d seconds" % (
+                timediff.years, timediff.months, timediff.days, timediff.hours, timediff.minutes, timediff.seconds))
+
+
+        if(stopThread == True):
+            print("closing Thread")
+            return
+
+    outputtext = outputtext + "Reached end of list.\nPress Exit to close the application." + "\n"
+    mainText.config(text=outputtext)
+    print("closing Thread")
+    return
+
+
+def save_information_to_addresslist(adress, adress_list, columnIndexes, coordinates, file_split_char, list_index,
+                                    image_filename, pv_Production100, roof_area, suitability, url,
+                                    value_electricity_production):
+    # Save Information in Addresslist
+    adress[columnIndexes[4]] = str(coordinates[0])
+    adress[columnIndexes[5]] = str(coordinates[1])
+    adress[columnIndexes[6]] = url
+    adress[columnIndexes[7]] = suitability
+    adress[columnIndexes[8]] = image_filename
+    adress[columnIndexes[9]] = pv_Production100
+    adress[columnIndexes[10]] = value_electricity_production
+    adress[columnIndexes[11]] = roof_area
+    new_line_string = ""
+    for j in adress:
+        new_line_string = new_line_string + str(j) + file_split_char
+    adress_list[list_index] = new_line_string
+
+
+def save_addresslist_in_file(adress_list, filename_adresslist):
+    new_adress_list = ""
+    for j in adress_list:
+        new_adress_list = new_adress_list + (j) + "\n"
+    # save File
+    adress_file = open(filename_adresslist, "w", encoding="utf-8")
+    adress_file.write(new_adress_list)
+    adress_file.close()
+
+def create_Partner_in_dolibarr(street, plz, city, coordinates, sumEnergyRoofs, sumRoofArea, suitability, featureId, url, roofs):
+    global electricityPrice
+    global dolibarrStateID
+    global dolibarrCountryID
+    global dolibarrCountryCode
+    global dolibarrURL
+    global dolibarrApiKey
+    params = {
+        "name": "unbekannt",
+        "address": street,
+        "zip": plz,
+        "town": city,
+        "state_id": dolibarrStateID,
+        "client": "2",
+        "code_client": "FID-" + featureId,
+        "note_public": "",
+        "status_prospect_label": "Jamais contacté",
+        "array_options": {
+            "options_energy": str(sumEnergyRoofs),
+            "options_energy_value": str(float(sumEnergyRoofs) * electricityPrice),
+            "options_roof_size": str(sumRoofArea),
+            "options_avrg_suitability": str(suitability),
+            "options_geomx": str(coordinates[0]),
+            "options_geomy": str(coordinates[1]),
+            "options_roof_info": create_roof_info(roofs),
+            "options_sonnendach_url": url
+        },
+        "country_id": dolibarrCountryID,
+        "country_code": dolibarrCountryCode
+        }
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "DOLAPIKEY": dolibarrApiKey
+    }
+    response = json.loads(requests.post(dolibarrURL, json=params, headers=headers).content.decode())
+    # print(params)
+    # print(response)
+
+def create_roof_info(roofs):
+    info_text = ""
+    for i in range(len(roofs)):
+        info_text = info_text + str(i + 1) + ". Fläche\n" + "Potenzielle Energie: " + str(roofs[i]['attributes']['gstrahlung']*0.2*0.8) + "\nFläche: " + str(roofs[i]['attributes']['flaeche']) + "\nEignung: " + str(roofs[i]['attributes']['klasse']) + " (" + str(roofs[i]['attributes']['klasse_text']).split("##")[0] + ")\n\n"
+    return info_text
+
+def create_images(address_string, driver, image_filename, image_folder_map, image_folder_production, image_folder_qrcode, url):
+    driver.get(url)
     time.sleep(2)
-                url = driver.current_url
-                suitability = driver.find_element(By.ID, "eignung").text
-                pv_Production50 = driver.find_element(By.ID, "pv50").text.replace("'", "")
-                pv_Production75 = driver.find_element(By.ID, "pv75").text.replace("'", "")
-                pv_Production100 = driver.find_element(By.ID, "pv100").text.replace("'", "")
-                value_electricity_production = driver.find_elements(By.XPATH, "//h2[@id='TitelSolarstrom']//strong")[2].text.replace("'", "").replace(" Franken", "")
-
-
-                image_filename = suitability + " - " + search_string
-                image_folder_map = "screenshots/"
-                image_filename_map = image_filename + " map" + ".png"
-                image_folder_production = "screenshots/"
-                image_filename_production = image_filename + " production" + ".png"
-                image_folder_qrcode = "qrcodes/"
-                image_filename_qrcode = image_filename + " qrcode" + ".png"
 
     # Create QR-Code
     qr = qrcode.QRCode(version=1, box_size=10, border=5)
     qr.add_data(url)
     qr.make(fit=True)
-                qr.make_image(fill='black', back_color='white').save(image_folder_qrcode + image_filename_qrcode)
+    qr.make_image(fill='black', back_color='white').save(image_folder_qrcode + image_filename)
 
     # Create Screenshot of Map
     actions = ActionChains(driver)
@@ -106,10 +319,10 @@ def search_adresses(adress_list, filename_adresslist, driver, mainText):
     area = (x, 0, w, h)
     driver.execute_script("window.scrollTo(0, " + str(y) + ")")
     time.sleep(1)
-                driver.save_screenshot(image_folder_map + image_filename_map)
+    driver.save_screenshot(image_folder_map + image_filename)
     time.sleep(0.2)
-                Image.open(image_folder_map + image_filename_map).crop(area).save(
+    Image.open(image_folder_map + image_filename).crop(area).save(
-                    image_folder_map + image_filename_map)
+        image_folder_map + image_filename)
 
     # Create Screenshot of Chart
     try:
@@ -124,59 +337,12 @@ def search_adresses(adress_list, filename_adresslist, driver, mainText):
         driver.execute_script("window.scrollTo(0, " + str(y) + ")")
 
         time.sleep(1)
-                    driver.save_screenshot(image_folder_production + image_filename_production)
+        driver.save_screenshot(image_folder_production + image_filename)
         time.sleep(0.2)
-                    Image.open(image_folder_production + image_filename_production).crop(area).save(
+        Image.open(image_folder_production + image_filename).crop(area).save(
-                        image_folder_production + image_filename_production)
+            image_folder_production + image_filename)
     except:
-                    image_filename_production = "not-found"
+        pass
-
-                print(image_filename_map + " was saved.")
-            else:
-                url = "not-found"
-                suitability = "not-found"
-                image_filename_map = "not-found"
-                image_filename_production = "not-found"
-                image_filename_qrcode = "not-found"
-                pv_Production50 = "not-found"
-                pv_Production75 = "not-found"
-                pv_Production100 = "not-found"
-                value_electricity_production = "not-found"
-
-
-                print("not found: " + search_string)
-
-
-            #Write back into file
-            adress[columnIndexes[4]] = url
-            adress[columnIndexes[5]] = suitability
-            adress[columnIndexes[6]] = image_filename_map
-            adress[columnIndexes[7]] = image_filename_production
-            adress[columnIndexes[8]] = image_filename_qrcode
-            adress[columnIndexes[9]] = pv_Production50
-            adress[columnIndexes[10]] = pv_Production75
-            adress[columnIndexes[11]] = pv_Production100
-            adress[columnIndexes[12]] = value_electricity_production
-            adress_file = open(filename_adresslist, "w", encoding="utf-8")
-            new_line_string = ""
-            for j in adress:
-                new_line_string = new_line_string + j + file_split_char
-            adress_list[i] = new_line_string
-            new_adress_list = ""
-            for j in adress_list:
-                new_adress_list = new_adress_list + (j) + "\n"
-            adress_file.write(new_adress_list)
-            adress_file.close()
-
-
-        if(stopThread == True):
-            print("closing Thread")
-            return
-
-    outputtext = outputtext + "Reached end of list.\nPress Exit to close the application." + "\n"
-    mainText.config(text=outputtext)
-    print("closing Thread")
-    return
 
 
 def read_adresslist(filename_adresslist):
@@ -232,7 +398,7 @@ def command():
             thread_search_adresses.join()
         except:
             pass
-        print("Thread is closed")
+        print("Threads are closed")
         root.quit()
         print("Application is closed")
         driver.quit()
@@ -243,7 +409,7 @@ def command():
         outputtext = outputtext + "Searching File " + filename_adresslist + "\n"
         mainText.config(text=outputtext)
         adress_list_result = read_adresslist(filename_adresslist)
-        if(adress_list_result):
+        if(adress_list_result[0]):
             adresslist = adress_list_result[1]
             outputtext = outputtext + "File " + filename_adresslist + " found." + "\n"
             mainText.config(text=outputtext)
@@ -297,7 +463,6 @@ def command_exit():
 
 
 
-
 root = tkinter.Tk()
 root.protocol("WM_DELETE_WINDOW", command_exit)
 root.wm_title("Sonnendach")
@@ -312,6 +477,9 @@ for i in range(len(DropDownLabels)):
 
 frameButtons = tkinter.Frame(root)
 frameButtons.grid(row=3, column=1)
+createImages = tkinter.IntVar()
+checkBoxCreateImages = tkinter.Checkbutton(frameButtons, text="create images", variable=createImages)
+checkBoxCreateImages.grid()
 button1 = tkinter.Button(frameButtons, text="Select Adresslist file", command=command, width=20, height=2, bg="#FCCA03")
 button1.grid(row=3, column=1, padx=10, pady=3)
 button2 = tkinter.Button(frameButtons, text="EXIT", command=command_exit, width=20, height=2, bg="#FCCA03")

api-Test.py

@@ -0,0 +1,57 @@
+#take a roof knowing the coordinates and use it to retrieve all the others "on the same building"
+import json
+import requests
+
+search_string = "Toggenburgstrasse 31 8245 Feuerthalen"
+
+#Address to coordinates
+params={
+"type":"locations",
+"searchText":search_string,
+"lang":"de"}
+locationsResponse = requests.get('https://api3.geo.admin.ch/rest/services/api/SearchServer', params=params)
+locations = json.loads(locationsResponse.content.decode())
+firstLocation = locations['results'][0]['attrs']
+
+# you need the coordinates from other services, once you have them
+coordinates=(firstLocation['y'],firstLocation['x']) # example coordinates
+coordinates_str=','.join(map(str,coordinates))
+tolerance=1
+
+# this is wrong, bbox should be computed using tolerance and pixel size, but there are no problems for the
+# purpose of this exercise
+bbox=(coordinates[0]-tolerance, coordinates[1]-tolerance, coordinates[0]+tolerance,coordinates[1]+tolerance)
+bbox_str=','.join(map(str,bbox))
+# retrieve one roof and the building_id associated
+params={
+"geometryType":"esriGeometryPoint",
+"returnGeometry":"true",
+"layers":"all:ch.bfe.solarenergie-eignung-daecher",
+"geometry":coordinates_str,
+"mapExtent":bbox_str,
+"imageDisplay":"1391,1070,96",
+"tolerance":tolerance,
+"order":"distance",
+"lang":"de",
+}
+print(coordinates_str)
+response_one_roof = requests.get('https://api3.geo.admin.ch/rest/services/api/MapServer/identify', params=params)
+response_one_roof_decoded = json.loads(response_one_roof.content.decode())
+results = (response_one_roof_decoded['results'])
+for result in results:
+    print(result['attributes']['building_id'])
+building_id=(response_one_roof_decoded)['results'][0]['attributes']['building_id']
+
+# use the building_id of previous request to get all roofs associated with this building_id
+params = {
+'layer': 'ch.bfe.solarenergie-eignung-daecher',
+'searchField': 'building_id',
+'contains': 'false',
+'searchText': str(building_id),
+'returnGeometry': 'false'
+}
+response_all_roofs = requests.get('https://api3.geo.admin.ch/rest/services/api/MapServer/find', params=params)
+response_all_roofs_decoded = json.loads(response_all_roofs.content.decode())['results']
+for roof in response_all_roofs_decoded:
+    # print(roof['attributes'].keys())
+    print("URL:", "https://www.uvek-gis.admin.ch/BFE/sonnendach/index.html?featureId=" + str(roof['featureId']) + "&lang=de", "Fläche:", roof['attributes']['flaeche'], "Eignung(1-5):", roof['attributes']['klasse'], "PV100:", float(roof['attributes']['gstrahlung']*0.2*0.8))