tools/assemblee_nationale/load_datas.py

# encoding: utf-8
"""
Tool used to upload representatives from French National Assembly.
"""

import os
import shutil

import requests

url = "https://data.assemblee-nationale.fr/static/openData/repository/15/amo/tous_acteurs_mandats_organes_xi_legislature/AMO30_tous_acteurs_tous_mandats_tous_organes_historique.json.zip"

data_root = os.environ.get("POLITIKORAMA_DATA_ROOT", "../tmp")
data_source = os.path.join(data_root, "assemblee_nationale.zip")
data_target = os.path.join(data_root, "json")

# Cleaning old data
try:
    os.remove(data_source)
except FileNotFoundError:
    # No file to remove
    pass
try:
    shutil.rmtree(data_target)
except FileNotFoundError:
    # No folder to remove
    pass

# Download and extract data
print("Downloading archive")
with requests.get(url, stream=True) as result:
    result.raise_for_status()
    with open(data_source, "wb") as f:
        for chunk in result.iter_content(chunk_size=8192):
            f.write(chunk)
print("Unpacking archive")
shutil.unpack_archive(data_source, data_root)

os.remove(data_source)