#Import Libraries
import numpy as np
import pandas as pd
import geopandas as gpd
import json
import requests
import math
import os
import time

# Import Folders
data_folder = os.path.abspath("data")
output_folder = os.path.abspath("output")
#check if outfolder exists if not create it
if not os.path.exists(output_folder):
    os.makedirs(output_folder)

# Merge all the scraped files and write a shapfile
layer_folders = os.listdir(output_folder)
layer_folders = layer_folders#[1:]
for layer in layer_folders:
    print("--Combining files for folder:", layer)
    # get list of all the json files
    layer_folder = os.path.join(output_folder, layer)
    files = os.listdir(layer_folder)
    files = [x for x in files if x.split(".")[-1] == "json"]
    print("---Total files to combine", len(files))
    # Combine all the files
    df_list = []
    for file in files:
        filepath = os.path.join(layer_folder, file)
        df1 = gpd.read_file(filepath)
        df_list.append(df1)
    ## Merge all files
    df = None
    df = pd.concat(df_list)
    #finaly save scrapped data as shapefile
    out_file = os.path.join(layer_folder, layer + ".shp")
    df.to_file(driver = 'ESRI Shapefile', filename = out_file)
    print("---Combined all files Finished file saved at: ", out_file)

--Combining files for folder: Counties (NUTS IV)
---Total files to combine 1
---Combined all files Finished file saved at:  C:\Users\sandyjones\Desktop\Scrape GIS Data\GIS Tools\output\Counties (NUTS IV)\Counties (NUTS IV).shp
--Combining files for folder: Electoral Divisions (NUTS V)
---Total files to combine 1
---Combined all files Finished file saved at:  C:\Users\sandyjones\Desktop\Scrape GIS Data\GIS Tools\output\Electoral Divisions (NUTS V)\Electoral Divisions (NUTS V).shp
--Combining files for folder: Gaeltacht Areas
---Total files to combine 1
---Combined all files Finished file saved at:  C:\Users\sandyjones\Desktop\Scrape GIS Data\GIS Tools\output\Gaeltacht Areas\Gaeltacht Areas.shp
--Combining files for folder: Provinces
---Total files to combine 1
---Combined all files Finished file saved at:  C:\Users\sandyjones\Desktop\Scrape GIS Data\GIS Tools\output\Provinces\Provinces.shp
--Combining files for folder: Regional Assemblies (NUTS II)
---Total files to combine 1
---Combined all files Finished file saved at:  C:\Users\sandyjones\Desktop\Scrape GIS Data\GIS Tools\output\Regional Assemblies (NUTS II)\Regional Assemblies (NUTS II).shp
--Combining files for folder: Regional Authorities (NUTS III)
---Total files to combine 1
---Combined all files Finished file saved at:  C:\Users\sandyjones\Desktop\Scrape GIS Data\GIS Tools\output\Regional Authorities (NUTS III)\Regional Authorities (NUTS III).shp