from bs4 import BeautifulSoup
import requests

soup = BeautifulSoup(requests.get('http://elections.interieur.gouv.fr/presidentielle-2017/').text, 'html.parser')

map_tag = soup.find('map')

area_tags = map_tag.find_all('area')

links = [tag.attrs['href'] for tag in area_tags]

unique_links = set(links)

def extract_candidate_votes(url):
    "Extracts votes for all candidates from a given url."
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'html.parser')
    # département name
    dept = soup.find(class_='span12 pub-fil-ariane').find_all('a')[-1].text
    # candidate votes
    table = soup.find('tbody')
    results = {}
    for row in table.find_all('tr'):
        results[row.td.text] = int(row.td.next_sibling.next_sibling.text.replace(' ', ''))
    # other data: non-voters, nil, invalid
    second_table = soup.find_all('tbody')[1]
    for row in second_table.find_all('tr'):
        if row.td.text in ['Blancs', 'Nuls', 'Abstentions']:
            results[row.td.text] = int(row.td.next_sibling.next_sibling.text.replace(' ', ''))
    return dept, results

url = "http://elections.interieur.gouv.fr/presidentielle-2017/" + list(unique_links)[0]

depts = []
results = []
for link in unique_links:
    dept, result = extract_candidate_votes("http://elections.interieur.gouv.fr/presidentielle-2017/" + link) 
    depts.append(dept)
    results.append(result)

import pandas as pd

df_votes = pd.DataFrame(results, index=depts)

%matplotlib inline
import matplotlib.pyplot as plt
plt.style.use('seaborn-talk')

vote_sum = df_votes.sum(axis=1).sort_values(ascending=True)
df_votes['total'] = vote_sum
df_votes = df_votes.sort_values(by='total', ascending=True)
df_votes = df_votes.drop('total', 1)

import shapefile

r = shapefile.Reader(r"files/departement_shapes/DEPARTEMENT.shp")
shapes = r.shapes()
records = r.records()
fields = r.fields

import numpy as np
from matplotlib.collections import LineCollection

def draw_departements_and_centroids(ax, records, shapes, radii=None):
    "Draws a map of the départements."
    if radii is None:
        radii = [10000 for _ in range(len(records))]
    for record, shape, radius in zip(records, shapes, radii):
        lons, lats = zip(*shape.points)
        data = np.array([lons, lats]).T

        if len(shape.parts) == 1:
            segs = [data,]
        else:
            segs = []
            for i in range(1,len(shape.parts)):
                index = shape.parts[i-1]
                index2 = shape.parts[i]
                segs.append(data[index:index2])
            segs.append(data[index2:])

        lines = LineCollection(segs,antialiaseds=(1,))
        lines.set_edgecolors('k')
        lines.set_linewidth(0.3)
        ax.add_collection(lines)
        
        centroid = data.mean(axis=0)
        ax.add_artist(plt.Circle(centroid, radius=radius, alpha=0.7))

def get_radii(label_idx, iterable=df.CODE_DEPT):
    """Returns sorted radii by correct label index."""
    s = df_votes[df_votes.columns[label_idx]]
    radii = []
    for code_dpt in iterable:
        values = s[s.index.str.contains("({})".format(code_dpt), regex=False)].values
        radii.append(values)
    return np.array(radii)

import networkx as nx

code_to_idx = dict(zip(df.CODE_DEPT, np.arange(df.shape[0])))

idx_to_code = dict(zip(np.arange(df.shape[0]), df.CODE_DEPT))

centroids = np.array([np.array(shape.points).mean(axis=0) for shape in shapes])

from scipy.spatial import KDTree

kdtree = KDTree(centroids)

pop_dict = dict(zip(df.CODE_DEPT, [vote_sum[vote_sum.index.str.contains("({})".format(idx), regex=False)].values[0] for idx in df.CODE_DEPT]))

G=nx.Graph()
for dept_id, centroid in zip(df.CODE_DEPT, range(centroids.shape[0])):
    G.add_node(dept_id)
    node_id = code_to_idx[dept_id]
    dists, neighbors = kdtree.query(kdtree.data[node_id], k=7)
    for neighbor in neighbors[1:]:
        neighbor_id = idx_to_code[neighbor]
        weight = -(pop_dict[neighbor_id])
        G.add_edge(dept_id, neighbor_id, weight=weight)

initial_pos = dict(zip([idx_to_code[idx] for idx in range(centroids.shape[0])], centroids))

pos=nx.spring_layout(G, pos=initial_pos, fixed=['75'], iterations=2, k=1e9, weight='weight')

df_votes['abstention, nuls, blancs'] = df_votes.Abstentions + df_votes.Nuls + df_votes.Blancs

def get_radii_by_col(col_label, iterable=df.CODE_DEPT):
    """Returns sorted radii by correct label index."""
    s = df_votes[col_label]
    radii = []
    for code_dpt in iterable:
        values = s[s.index.str.contains("({})".format(code_dpt), regex=False)].values
        radii.append(values)
    return np.array(radii)