jmtrivial/agenda_culturel
4
4
Fork 0
Code Issues 66 Pull Requests 1 Projects 1 Releases Wiki Activity

* Ajout de l'import récurrent (manque la partie cron)

Browse Source 
* Correction des textes en français. Fix #73
This commit is contained in:
Jean-Marie Favreau 2024-01-26 13:35:47 +01:00
parent 2bac3f29b5
commit 59c091a2f5
27 changed files with 1063 additions and 477 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

252
experimentations/get_ical_events.py
View File

@ -1,249 +1,29 @@
#!/usr/bin/python3
# coding: utf-8
from abc import ABC, abstractmethod
from urllib.parse import urlparse
import urllib.request
import os
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
import icalendar
from icalendar import vDatetime
from datetime import datetime, date
import json
from bs4 import BeautifulSoup
import pickle
import sys
# getting the name of the directory
# where the this file is present.
current = os.path.dirname(os.path.realpath(__file__))
# Getting the parent directory name
# where the current directory is present.
parent = os.path.dirname(current)
# adding the parent directory to
# the sys.path.
sys.path.append(parent)
class Downloader(ABC):
def __init__(self):
pass
@abstractmethod
def download(self, url):
pass
class SimpleDownloader(Downloader):
def __init__(self):
super().__init__()
def download(self, url):
print("Downloading {}".format(url))
try:
resource = urllib.request.urlopen(url)
data = resource.read().decode(resource.headers.get_content_charset())
return data
except:
return None
from src.agenda_culturel.import_tasks.downloader import *
from src.agenda_culturel.import_tasks.extractor import *
from src.agenda_culturel.import_tasks.importer import *
from src.agenda_culturel.import_tasks.extractor_ical import *
class ChromiumHeadlessDownloader(Downloader):
def __init__(self):
super().__init__()
options = Options()
options.add_argument("--headless=new")
service = Service("/usr/bin/chromedriver")
self.driver = webdriver.Chrome(service=service, options=options)
def download(self, url):
print("Download {}".format(url))
self.driver.get(url)
return driver.page_source
class Extractor(ABC):
def __init__(self):
self.header = {}
self.events = []
@abstractmethod
def extract(self, content, url, url_human = None):
pass
def set_header(self, url):
self.header["url"] = url
self.header["date"] = datetime.now()
def clear_events(self):
self.events = []
def add_event(self, title, category, start_day, location, description, tags, uuid, recurrences=None, url_human=None, start_time=None, end_day=None, end_time=None, last_modified=None, published=False):
if title is None:
print("ERROR: cannot import an event without name")
return
if start_day is None:
print("ERROR: cannot import an event without start day")
return
event = {
"title": title,
"category": category,
"start_day": start_day,
"uuid": uuid,
"location": location,
"description": description,
"tags": tags,
"published": published
}
if url_human is not None:
event["url_human"] = url_human
if start_time is not None:
event["start_time"] = start_time
if end_day is not None:
event["end_day"] = end_day
if end_time is not None:
event["end_time"] = end_time
if last_modified is not None:
event["last_modified"] = last_modified
if recurrences is not None:
event["recurrences"] = recurrences
self.events.append(event)
def default_value_if_exists(self, default_values, key):
return default_values[key] if default_values is not None and key in default_values else None
def get_structure(self):
return { "header": self.header, "events": self.events}
class ICALExtractor(Extractor):
def __init__(self):
super().__init__()
def get_item_from_vevent(self, event, name, raw = False):
try:
r = event.decoded(name)
if raw:
return r
else:
return r.decode()
except:
return None
def get_dt_item_from_vevent(self, event, name):
item = self.get_item_from_vevent(event, name, raw = True)
day = None
time = None
if item is not None:
if isinstance(item, datetime):
day = item.date()
time = item.time()
elif isinstance(item, date):
day = item
time = None
return day, time
def extract(self, content, url, url_human = None, default_values = None, published = False):
print("Extracting ical events from {}".format(url))
self.set_header(url)
self.clear_events()
self.uuids = {}
calendar = icalendar.Calendar.from_ical(content)
for event in calendar.walk('VEVENT'):
title = self.get_item_from_vevent(event, "SUMMARY")
category = self.default_value_if_exists(default_values, "category")
start_day, start_time = self.get_dt_item_from_vevent(event, "DTSTART")
end_day, end_time = self.get_dt_item_from_vevent(event, "DTEND")
location = self.get_item_from_vevent(event, "LOCATION")
if location is None:
location = self.default_value_if_exists(default_values, "location")
description = self.get_item_from_vevent(event, "DESCRIPTION")
if description is not None:
soup = BeautifulSoup(description)
delimiter = '\n'
for line_break in soup.findAll('br'):
line_break.replaceWith(delimiter)
description = soup.get_text()
last_modified = self.get_item_from_vevent(event, "LAST_MODIFIED")
uuid = self.get_item_from_vevent(event, "UID")
if uuid is not None:
if uuid in self.uuids:
self.uuids[uuid] += 1
uuid += ":{:04}".format(self.uuids[uuid] - 1)
else:
self.uuids[uuid] = 1
event_url = url + "#" + uuid
tags = self.default_value_if_exists(default_values, "tags")
last_modified = self.get_item_from_vevent(event, "LAST-MODIFIED", raw = True)
recurrence_entries = {}
for e in ["RRULE", "EXRULE", "EXDATE", "RDATE"]:
i = self.get_item_from_vevent(event, e, raw = True)
if i is not None:
recurrence_entries[e] = i
if start_day is not None and len(recurrence_entries) != 0:
recurrences = ""
for k, r in recurrence_entries.items():
if isinstance(r, list):
recurrences += "\n".join([k + ":" + e.to_ical().decode() for e in r]) + "\n"
else:
recurrences += k + ":" + r.to_ical().decode() + "\n"
else:
recurrences = None
self.add_event(title, category, start_day, location, description, tags, recurrences=recurrences, uuid=event_url, url_human=url_human, start_time=start_time, end_day=end_day, end_time=end_time, last_modified=last_modified, published=published)
return self.get_structure()
class URL2Events:
def __init__(self, downloader, extractor):
self.downloader = downloader
self.extractor = extractor
def process(self, url, url_human = None, cache = None, default_values = None, published = False):
if cache and os.path.exists(cache):
print("Loading cache ({})".format(cache))
with open(cache) as f:
content = "\n".join(f.readlines())
else:
content = self.downloader.download(url)
if cache:
print("Saving cache ({})".format(cache))
dir = os.path.dirname(cache)
if dir != "" and not os.path.exists(dir):
os.makedirs(dir)
with open(cache, "w") as text_file:
text_file.write(content)
return self.extractor.extract(content, url, url_human, default_values, published)
if __name__ == "__main__":

3
src/agenda_culturel/admin.py
View File

@ -1,6 +1,6 @@
from django.contrib import admin
from django import forms
from .models import Event, Category, StaticContent, DuplicatedEvents, BatchImportation
from .models import Event, Category, StaticContent, DuplicatedEvents, BatchImportation, RecurrentImport
from django_better_admin_arrayfield.admin.mixins import DynamicArrayMixin
from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget
from django_better_admin_arrayfield.models.fields import DynamicArrayField
@ -10,6 +10,7 @@ admin.site.register(Category)
admin.site.register(StaticContent)
admin.site.register(DuplicatedEvents)
admin.site.register(BatchImportation)
admin.site.register(RecurrentImport)
class URLWidget(DynamicArrayWidget):

74
src/agenda_culturel/celery.py
View File

@ -1,4 +1,5 @@
import os
import json
from celery import Celery
from celery.schedules import crontab
@ -6,6 +7,13 @@ from celery.utils.log import get_task_logger
from .extractors import ExtractorAllURLs
from .import_tasks.downloader import *
from .import_tasks.extractor import *
from .import_tasks.importer import *
from .import_tasks.extractor_ical import *
# Set the default Django settings module for the 'celery' program.
APP_ENV = os.getenv("APP_ENV", "dev")
@ -44,14 +52,20 @@ def close_import_task(taskid, success, error_message, importer):
@app.task(bind=True)
def import_events_from_json(self, json):
from agenda_culturel.models import Event
from agenda_culturel.models import Event, BatchImportation
from .importation import EventsImporter
# create a batch importation
importation = BatchImportation(celery_id=self.request.id)
# save batch importation
importation.save()
logger.info("Import events from json: {}".format(self.request.id))
importer = EventsImporter(self.request.id)
# try:
#try:
success, error_message = importer.import_events(json)
# finally, close task
@ -62,25 +76,55 @@ def import_events_from_json(self, json):
@app.task(bind=True)
def import_events_from_url(self, source, browsable_url):
from agenda_culturel.models import Event
def run_recurrent_import(self, pk):
from agenda_culturel.models import RecurrentImport, BatchImportation
from .importation import EventsImporter
from django.shortcuts import get_object_or_404
logger.info("Import events from url: {} {}".format(source, self.request.id))
# first get json
# TODO
logger.info("Run recurrent import: {}".format(self.request.id))
# then import
# importer = EventsImporter(self.request.id)
# success, error_message = importer.import_events(json)
# get the recurrent import
rimport = RecurrentImport.objects.get(pk=pk)
# create a batch importation
importation = BatchImportation(recurrentImport=rimport, celery_id=self.request.id)
# save batch importation
importation.save()
success = True
error_message = ""
# create an importer
importer = EventsImporter(self.request.id)
# finally, close task
close_import_task(self.request.id, success, error_message)
# prepare downloading and extracting processes
downloader = SimpleDownloader() if rimport.downloader == RecurrentImport.DOWNLOADER.SIMPLE else ChromiumHeadlessDownloader()
extractor = ICALExtractor() if rimport.processor == RecurrentImport.PROCESSOR.ICAL else None
if extractor is None:
logger.error("Unknown extractor")
close_import_task(self.request.id, False, "Unknown extractor", importer)
# set parameters
u2e = URL2Events(downloader, extractor)
url = rimport.source
browsable_url = rimport.browsable_url
category = str(rimport.defaultCategory)
location = rimport.defaultLocation
tags = rimport.defaultTags
try:
# get events from website
events = u2e.process(url, browsable_url, default_values = {"category": category, "location": location, "tags": tags}, published = True)
# convert it to json
json_events = json.dumps(events, default=str)
# import events (from json)
success, error_message = importer.import_events(json_events)
# finally, close task
close_import_task(self.request.id, success, error_message, importer)
except Exception as e:
logger.error(e)
close_import_task(self.request.id, False, e, importer)

29
src/agenda_culturel/forms.py
View File

@ -2,7 +2,7 @@ from django.forms import ModelForm, ValidationError, TextInput, Form, URLField,
from datetime import date
from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget
from .models import Event, BatchImportation
from .models import Event, BatchImportation, RecurrentImport
from django.utils.translation import gettext_lazy as _
from string import ascii_uppercase as auc
from .templatetags.utils_extra import int_to_abc
@ -25,6 +25,11 @@ class DynamicArrayWidgetURLs(DynamicArrayWidget):
class DynamicArrayWidgetTags(DynamicArrayWidget):
template_name = "agenda_culturel/widgets/widget-tags.html"
class RecurrentImportForm(ModelForm):
class Meta:
model = RecurrentImport
fields = '__all__'
class EventForm(ModelForm):
class Meta:
@ -75,26 +80,8 @@ class EventForm(ModelForm):
class BatchImportationForm(ModelForm):
json = CharField(label="JSON (facultatif)", widget=Textarea(attrs={"rows":"10"}), help_text=_("JSON in the format expected for the import. If the JSON is provided here, we will ignore the URLs given above, and use the information provided by the json without importing any additional events from the URL."), required=False)
class Meta:
model = BatchImportation
fields = ['source', 'browsable_url']
def clean(self):
cleaned_data = super().clean()
json = cleaned_data.get("json")
source = cleaned_data.get("source")
browsable_url = cleaned_data.get("browsable_url")
if (not json or json == "") and (not source or source == "") and (not browsable_url or browsable_url == ""):
raise ValidationError(_("You need to fill in either the json or the source possibly supplemented by the navigable URL."))
# Always return a value to use as the new cleaned data, even if
# this method didn't change it.
return cleaned_data
class BatchImportationForm(Form):
json = CharField(label="JSON", widget=Textarea(attrs={"rows":"10"}), help_text=_("JSON in the format expected for the import."), required=True)
class FixDuplicates(Form):

50
src/agenda_culturel/import_tasks/downloader.py Normal file
View File

@ -0,0 +1,50 @@
from urllib.parse import urlparse
import urllib.request
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from abc import ABC, abstractmethod
class Downloader(ABC):
def __init__(self):
pass
@abstractmethod
def download(self, url):
pass
class SimpleDownloader(Downloader):
def __init__(self):
super().__init__()
def download(self, url):
print("Downloading {}".format(url))
try:
resource = urllib.request.urlopen(url)
data = resource.read().decode(resource.headers.get_content_charset())
return data
except:
return None
class ChromiumHeadlessDownloader(Downloader):
def __init__(self):
super().__init__()
options = Options()
options.add_argument("--headless=new")
service = Service("/usr/bin/chromedriver")
self.driver = webdriver.Chrome(service=service, options=options)
def download(self, url):
print("Download {}".format(url))
self.driver.get(url)
return driver.page_source

62
src/agenda_culturel/import_tasks/extractor.py Normal file
View File

@ -0,0 +1,62 @@
from abc import ABC, abstractmethod
from bs4 import BeautifulSoup
from datetime import datetime
class Extractor(ABC):
def __init__(self):
self.header = {}
self.events = []
@abstractmethod
def extract(self, content, url, url_human = None):
pass
def set_header(self, url):
self.header["url"] = url
self.header["date"] = datetime.now()
def clear_events(self):
self.events = []
def add_event(self, title, category, start_day, location, description, tags, uuid, recurrences=None, url_human=None, start_time=None, end_day=None, end_time=None, last_modified=None, published=False):
if title is None:
print("ERROR: cannot import an event without name")
return
if start_day is None:
print("ERROR: cannot import an event without start day")
return
event = {
"title": title,
"category": category,
"start_day": start_day,
"uuid": uuid,
"location": location,
"description": description,
"tags": tags,
"published": published
}
if url_human is not None:
event["url_human"] = url_human
if start_time is not None:
event["start_time"] = start_time
if end_day is not None:
event["end_day"] = end_day
if end_time is not None:
event["end_time"] = end_time
if last_modified is not None:
event["last_modified"] = last_modified
if recurrences is not None:
event["recurrences"] = recurrences
self.events.append(event)
def default_value_if_exists(self, default_values, key):
return default_values[key] if default_values is not None and key in default_values else None
def get_structure(self):
return { "header": self.header, "events": self.events}

110
src/agenda_culturel/import_tasks/extractor_ical.py Normal file
View File

@ -0,0 +1,110 @@
import icalendar
import warnings
from icalendar import vDatetime
from datetime import datetime, date
from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
from .extractor import *
class ICALExtractor(Extractor):
def __init__(self):
super().__init__()
def get_item_from_vevent(self, event, name, raw = False):
try:
r = event.decoded(name)
if raw:
return r
else:
return r.decode()
except:
return None
def get_dt_item_from_vevent(self, event, name):
item = self.get_item_from_vevent(event, name, raw = True)
day = None
time = None
if item is not None:
if isinstance(item, datetime):
day = item.date()
time = item.time()
elif isinstance(item, date):
day = item
time = None
return day, time
def extract(self, content, url, url_human = None, default_values = None, published = False):
warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
print("Extracting ical events from {}".format(url))
self.set_header(url)
self.clear_events()
self.uuids = {}
calendar = icalendar.Calendar.from_ical(content)
for event in calendar.walk('VEVENT'):
title = self.get_item_from_vevent(event, "SUMMARY")
category = self.default_value_if_exists(default_values, "category")
start_day, start_time = self.get_dt_item_from_vevent(event, "DTSTART")
end_day, end_time = self.get_dt_item_from_vevent(event, "DTEND")
location = self.get_item_from_vevent(event, "LOCATION")
if location is None:
location = self.default_value_if_exists(default_values, "location")
description = self.get_item_from_vevent(event, "DESCRIPTION")
if description is not None:
soup = BeautifulSoup(description, features="lxml")
delimiter = '\n'
for line_break in soup.findAll('br'):
line_break.replaceWith(delimiter)
description = soup.get_text()
last_modified = self.get_item_from_vevent(event, "LAST_MODIFIED")
uuid = self.get_item_from_vevent(event, "UID")
if uuid is not None:
if uuid in self.uuids:
self.uuids[uuid] += 1
uuid += ":{:04}".format(self.uuids[uuid] - 1)
else:
self.uuids[uuid] = 1
event_url = url + "#" + uuid
tags = self.default_value_if_exists(default_values, "tags")
last_modified = self.get_item_from_vevent(event, "LAST-MODIFIED", raw = True)
recurrence_entries = {}
for e in ["RRULE", "EXRULE", "EXDATE", "RDATE"]:
i = self.get_item_from_vevent(event, e, raw = True)
if i is not None:
recurrence_entries[e] = i
if start_day is not None and len(recurrence_entries) != 0:
recurrences = ""
for k, r in recurrence_entries.items():
if isinstance(r, list):
recurrences += "\n".join([k + ":" + e.to_ical().decode() for e in r]) + "\n"
else:
recurrences += k + ":" + r.to_ical().decode() + "\n"
else:
recurrences = None
if title is not None:
self.add_event(title, category, start_day, location, description, tags, recurrences=recurrences, uuid=event_url, url_human=url_human, start_time=start_time, end_day=end_day, end_time=end_time, last_modified=last_modified, published=published)
return self.get_structure()

31
src/agenda_culturel/import_tasks/importer.py Normal file
View File

@ -0,0 +1,31 @@
import os
from .downloader import *
from .extractor import *
class URL2Events:
def __init__(self, downloader, extractor):
self.downloader = downloader
self.extractor = extractor
def process(self, url, url_human = None, cache = None, default_values = None, published = False):
if cache and os.path.exists(cache):
print("Loading cache ({})".format(cache))
with open(cache) as f:
content = "\n".join(f.readlines())
else:
content = self.downloader.download(url)
if cache:
print("Saving cache ({})".format(cache))
dir = os.path.dirname(cache)
if dir != "" and not os.path.exists(dir):
os.makedirs(dir)
with open(cache, "w") as text_file:
text_file.write(content)
return self.extractor.extract(content, url, url_human, default_values, published)

352
src/agenda_culturel/locale/fr/LC_MESSAGES/django.po
View File

@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: agenda_culturel\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-12-29 15:35+0000\n"
"POT-Creation-Date: 2024-01-26 12:27+0000\n"
"PO-Revision-Date: 2023-10-29 14:16+0000\n"
"Last-Translator: Jean-Marie Favreau <jeanmarie.favreau@free.fr>\n"
"Language-Team: Jean-Marie Favreau <jeanmarie.favreau@free.fr>\n"
@ -17,340 +17,395 @@ msgstr ""
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
#: agenda_culturel/forms.py:38
#: agenda_culturel/forms.py:62
msgid "The end date must be after the start date."
msgstr "La date de fin doit être après la date de début."
#: agenda_culturel/forms.py:53
#: agenda_culturel/forms.py:77
msgid "The end time cannot be earlier than the start time."
msgstr "L'heure de fin ne peut pas être avant l'heure de début."
#: agenda_culturel/forms.py:61
msgid ""
"JSON in the format expected for the import. If the JSON is provided here, we "
"will ignore the URLs given above, and use the information provided by the "
"json without importing any additional events from the URL."
msgstr ""
"JSON au format attendu pour l'import. Si le JSON est fourni ici, on ignorera "
"les URL données au dessus, et on utilisera les informations fournies par le "
"json sans réaliser d'importation supplémentaire d'événements depuis l'URL."
#: agenda_culturel/forms.py:84
msgid "JSON in the format expected for the import."
msgstr "JSON dans le format attendu pour l'import"
#: agenda_culturel/forms.py:74
msgid ""
"You need to fill in either the json or the source possibly supplemented by "
"the navigable URL."
msgstr ""
"Vous devez renseigner soit le json soit la source éventuellement complétée "
"de l'URL navigable."
#: agenda_culturel/models.py:26 agenda_culturel/models.py:55
#: agenda_culturel/models.py:465
#: agenda_culturel/models.py:32 agenda_culturel/models.py:61
#: agenda_culturel/models.py:695
msgid "Name"
msgstr "Nom"
#: agenda_culturel/models.py:26 agenda_culturel/models.py:55
#: agenda_culturel/models.py:32 agenda_culturel/models.py:61
msgid "Category name"
msgstr "Nom de la catégorie"
#: agenda_culturel/models.py:27
#: agenda_culturel/models.py:33
msgid "Content"
msgstr "Contenu"
#: agenda_culturel/models.py:27
#: agenda_culturel/models.py:33
msgid "Text as shown to the visitors"
msgstr "Text tel que présenté aux visiteureuses"
#: agenda_culturel/models.py:28
#: agenda_culturel/models.py:34
msgid "URL path"
msgstr ""
#: agenda_culturel/models.py:28
#: agenda_culturel/models.py:34
msgid "URL path where the content is included."
msgstr ""
#: agenda_culturel/models.py:56
#: agenda_culturel/models.py:62
msgid "Alternative Name"
msgstr "Nom alternatif"
#: agenda_culturel/models.py:56
#: agenda_culturel/models.py:62
msgid "Alternative name used with a time period"
msgstr "Nom alternatif utilisé avec une période de temps"
#: agenda_culturel/models.py:57
#: agenda_culturel/models.py:63
msgid "Short name"
msgstr "Nom court"
#: agenda_culturel/models.py:57
#: agenda_culturel/models.py:63
msgid "Short name of the category"
msgstr "Nom court de la catégorie"
#: agenda_culturel/models.py:58
#: agenda_culturel/models.py:64
msgid "Color"
msgstr "Couleur"
#: agenda_culturel/models.py:58
#: agenda_culturel/models.py:64
msgid "Color used as background for the category"
msgstr "Couleur utilisée comme fond de la catégorie"
#: agenda_culturel/models.py:95 agenda_culturel/models.py:142
#: agenda_culturel/models.py:101 agenda_culturel/models.py:168
#: agenda_culturel/models.py:733
msgid "Category"
msgstr "Catégorie"
#: agenda_culturel/models.py:96
#: agenda_culturel/models.py:102
msgid "Categories"
msgstr "Catégories"
#: agenda_culturel/models.py:130
#: agenda_culturel/models.py:153
msgid "Published"
msgstr "Publié"
#: agenda_culturel/models.py:131
#: agenda_culturel/models.py:154
msgid "Draft"
msgstr "Brouillon"
#: agenda_culturel/models.py:132
#: agenda_culturel/models.py:155
msgid "Trash"
msgstr "Corbeille"
#: agenda_culturel/models.py:138
#: agenda_culturel/models.py:164
msgid "Title"
msgstr "Titre"
#: agenda_culturel/models.py:138
#: agenda_culturel/models.py:164
msgid "Short title"
msgstr "Titre court"
#: agenda_culturel/models.py:140 agenda_culturel/models.py:492
#: agenda_culturel/models.py:166 agenda_culturel/models.py:756
msgid "Status"
msgstr "Status"
#: agenda_culturel/models.py:142
#: agenda_culturel/models.py:168
msgid "Category of the event"
msgstr "Catégorie de l'événement"
#: agenda_culturel/models.py:144
#: agenda_culturel/models.py:170
msgid "Day of the event"
msgstr "Date de l'événement"
#: agenda_culturel/models.py:145
#: agenda_culturel/models.py:171
msgid "Starting time"
msgstr "Heure de début"
#: agenda_culturel/models.py:147
#: agenda_culturel/models.py:173
msgid "End day of the event"
msgstr "Fin de l'événement"
#: agenda_culturel/models.py:147
#: agenda_culturel/models.py:173
msgid "End day of the event, only required if different from the start day."
msgstr ""
"Date de fin de l'événement, uniquement nécessaire s'il est différent du "
"premier jour de l'événement"
#: agenda_culturel/models.py:148
#: agenda_culturel/models.py:174
msgid "Final time"
msgstr "Heure de fin"
#: agenda_culturel/models.py:150
#: agenda_culturel/models.py:176
msgid "Recurrence"
msgstr "Récurrence"
#: agenda_culturel/models.py:178 agenda_culturel/models.py:731
msgid "Location"
msgstr "Localisation"
#: agenda_culturel/models.py:150
#: agenda_culturel/models.py:178
msgid "Address of the event"
msgstr "Adresse de l'événement"
#: agenda_culturel/models.py:152
#: agenda_culturel/models.py:180
msgid "Description"
msgstr "Description"
#: agenda_culturel/models.py:152
#: agenda_culturel/models.py:180
msgid "General description of the event"
msgstr "Description générale de l'événement"
#: agenda_culturel/models.py:154
#: agenda_culturel/models.py:182
msgid "Illustration (local image)"
msgstr "Illustration (image locale)"
#: agenda_culturel/models.py:154
#: agenda_culturel/models.py:182
msgid "Illustration image stored in the agenda server"
msgstr "Image d'illustration stockée sur le serveur de l'agenda"
#: agenda_culturel/models.py:156
#: agenda_culturel/models.py:184
msgid "Illustration"
msgstr "Illustration"
#: agenda_culturel/models.py:156
#: agenda_culturel/models.py:184
msgid "URL of the illustration image"
msgstr "URL de l'image illustrative"
#: agenda_culturel/models.py:157
#: agenda_culturel/models.py:185
msgid "Illustration description"
msgstr "Description de l'illustration"
#: agenda_culturel/models.py:157
#: agenda_culturel/models.py:185
msgid "Alternative text used by screen readers for the image"
msgstr "Texte alternatif utiliser par les lecteurs d'écrans pour l'image"
#: agenda_culturel/models.py:159
#: agenda_culturel/models.py:187
msgid "Importation source"
msgstr "Source d'importation"
#: agenda_culturel/models.py:187
msgid "Importation source used to detect removed entries."
msgstr "Source d'importation utilisée pour détecter les éléments supprimés/"
#: agenda_culturel/models.py:188
msgid "UUIDs"
msgstr "UUIDs"
#: agenda_culturel/models.py:159
#: agenda_culturel/models.py:188
msgid "UUIDs from import to detect duplicated entries."
msgstr "UUIDs utilisés pendant l'import pour détecter les entrées dupliquées"
#: agenda_culturel/models.py:160
#: agenda_culturel/models.py:189
msgid "URLs"
msgstr "URLs"
#: agenda_culturel/models.py:160
#: agenda_culturel/models.py:189
msgid "List of all the urls where this event can be found."
msgstr "Liste de toutes les urls où l'événement peut être trouvé."
#: agenda_culturel/models.py:162
#: agenda_culturel/models.py:191
msgid "Tags"
msgstr "Étiquettes"
#: agenda_culturel/models.py:162
#: agenda_culturel/models.py:191
msgid "A list of tags that describe the event."
msgstr "Une liste d'étiquettes décrivant l'événement"
#: agenda_culturel/models.py:164
#: agenda_culturel/models.py:193
msgid "Possibly duplicated"
msgstr "Possibles doublons"
#: agenda_culturel/models.py:194
#: agenda_culturel/models.py:234
msgid "Event"
msgstr "Événement"
#: agenda_culturel/models.py:195
#: agenda_culturel/models.py:235
msgid "Events"
msgstr "Événements"
#: agenda_culturel/models.py:464
#: agenda_culturel/models.py:694
msgid "Subject"
msgstr "Sujet"
#: agenda_culturel/models.py:464
#: agenda_culturel/models.py:694
msgid "The subject of your message"
msgstr "Sujet de votre message"
#: agenda_culturel/models.py:465
#: agenda_culturel/models.py:695
msgid "Your name"
msgstr "Votre nom"
#: agenda_culturel/models.py:466
#: agenda_culturel/models.py:696
msgid "Email address"
msgstr "Adresse email"
#: agenda_culturel/models.py:466
#: agenda_culturel/models.py:696
msgid "Your email address"
msgstr "Votre adresse email"
#: agenda_culturel/models.py:467
#: agenda_culturel/models.py:697
msgid "Message"
msgstr "Message"
#: agenda_culturel/models.py:467
#: agenda_culturel/models.py:697
msgid "Your message"
msgstr "Votre message"
#: agenda_culturel/models.py:471 agenda_culturel/views.py:343
#: agenda_culturel/models.py:701 agenda_culturel/views.py:366
msgid "Closed"
msgstr "Fermé"
#: agenda_culturel/models.py:471
#: agenda_culturel/models.py:701
msgid "this message has been processed and no longer needs to be handled"
msgstr "Ce message a été traité et ne nécessite plus d'être pris en charge"
#: agenda_culturel/models.py:472
#: agenda_culturel/models.py:702
msgid "Comments"
msgstr "Commentaires"
#: agenda_culturel/models.py:472
#: agenda_culturel/models.py:702
msgid "Comments on the message from the moderation team"
msgstr "Commentaires sur ce message par l'équipe de modération"
#: agenda_culturel/models.py:481
msgid "Running"
msgstr ""
#: agenda_culturel/models.py:711
msgid "ical"
msgstr "ical"
#: agenda_culturel/models.py:482
msgid "Canceled"
msgstr "Annulé"
#: agenda_culturel/models.py:714
msgid "simple"
msgstr "simple"
#: agenda_culturel/models.py:483
msgid "Success"
msgstr "Succès"
#: agenda_culturel/models.py:715
msgid "Headless Chromium"
msgstr "chromium sans interface"
#: agenda_culturel/models.py:484
msgid "Failed"
msgstr "Erreur"
#: agenda_culturel/models.py:719
msgid "daily"
msgstr "chaque jour"
#: agenda_culturel/models.py:489
#: agenda_culturel/models.py:720
msgid "weekly"
msgstr "chaque semaine"
#: agenda_culturel/models.py:722
msgid "Processor"
msgstr "Processeur"
#: agenda_culturel/models.py:723
msgid "Downloader"
msgstr "Téléchargeur"
#: agenda_culturel/models.py:725
msgid "Import recurrence"
msgstr "Récurrence d'import"
#: agenda_culturel/models.py:728
msgid "Source"
msgstr "Source"
#: agenda_culturel/models.py:489
#: agenda_culturel/models.py:728
msgid "URL of the source document"
msgstr "URL du document source"
#: agenda_culturel/models.py:490
#: agenda_culturel/models.py:729
msgid "Browsable url"
msgstr "URL navigable"
#: agenda_culturel/models.py:490
#: agenda_culturel/models.py:729
msgid "URL of the corresponding document that will be shown to visitors."
msgstr "URL correspondant au document et qui sera montrée aux visiteurs"
#: agenda_culturel/models.py:494
#: agenda_culturel/models.py:731
msgid "Address for each imported event"
msgstr "Adresse de chaque événement importé"
#: agenda_culturel/models.py:733
msgid "Category of each imported event"
msgstr "Catégorie de chaque événement importé"
#: agenda_culturel/models.py:734
msgid "Tags for each imported event"
msgstr "Étiquettes de chaque événement importé"
#: agenda_culturel/models.py:734
msgid "A list of tags that describe each imported event."
msgstr "Une liste d'étiquettes décrivant chaque événement importé"
#: agenda_culturel/models.py:746
msgid "Running"
msgstr "En cours"
#: agenda_culturel/models.py:747
msgid "Canceled"
msgstr "Annulé"
#: agenda_culturel/models.py:748
msgid "Success"
msgstr "Succès"
#: agenda_culturel/models.py:749
msgid "Failed"
msgstr "Erreur"
#: agenda_culturel/models.py:754
msgid "Recurrent import"
msgstr "Import récurrent"
#: agenda_culturel/models.py:754
msgid "Reference to the recurrent import processing"
msgstr "Référence du processus d'import récurrent"
#: agenda_culturel/models.py:758
msgid "Error message"
msgstr "Votre message"
#: agenda_culturel/models.py:496
#: agenda_culturel/models.py:760
msgid "Number of collected events"
msgstr "Nombre d'événements collectés"
#: agenda_culturel/models.py:497
#: agenda_culturel/models.py:761
msgid "Number of imported events"
msgstr "Nombre d'événements importés"
#: agenda_culturel/models.py:498
#: agenda_culturel/models.py:762
msgid "Number of updated events"
msgstr "Nombre d'événements mis à jour"
#: agenda_culturel/models.py:499
#: agenda_culturel/models.py:763
msgid "Number of removed events"
msgstr "Nombre d'événements supprimés"
#: agenda_culturel/settings/base.py:134
#: agenda_culturel/settings/base.py:135
msgid "English"
msgstr "anglais"
#: agenda_culturel/settings/base.py:135
#: agenda_culturel/settings/base.py:136
msgid "French"
msgstr "français"
#: agenda_culturel/views.py:190
#: agenda_culturel/views.py:203
msgid "The static content has been successfully updated."
msgstr "Le contenu statique a été modifié avec succès."
#: agenda_culturel/views.py:196
#: agenda_culturel/views.py:209
msgid "The event has been successfully modified."
msgstr "L'événement a été modifié avec succès."
#: agenda_culturel/views.py:207
#: agenda_culturel/views.py:220
msgid "The event has been successfully deleted."
msgstr "L'événement a été supprimé avec succès"
#: agenda_culturel/views.py:224
#: agenda_culturel/views.py:247
msgid "The status has been successfully modified."
msgstr "Le status a été modifié avec succès."
#: agenda_culturel/views.py:246
#: agenda_culturel/views.py:269
msgid "The event is saved."
msgstr "L'événement est enregistré."
#: agenda_culturel/views.py:249
#: agenda_culturel/views.py:272
msgid ""
"The event has been submitted and will be published as soon as it has been "
"validated by the moderation team."
@ -358,7 +413,7 @@ msgstr ""
"L'événement a été soumis et sera publié dès qu'il aura été validé par "
"l'équipe de modération."
#: agenda_culturel/views.py:279
#: agenda_culturel/views.py:302
msgid ""
"The event has been successfully extracted, and you can now submit it after "
"modifying it if necessary."
@ -366,7 +421,7 @@ msgstr ""
"L'événement a été extrait avec succès, vous pouvez maintenant le soumettre "
"après l'avoir modifié au besoin."
#: agenda_culturel/views.py:283
#: agenda_culturel/views.py:306
msgid ""
"Unable to extract an event from the proposed URL. Please use the form below "
"to submit the event."
@ -374,12 +429,16 @@ msgstr ""
"Impossible d'extraire un événement depuis l'URL proposée. Veuillez utiliser "
"le formulaire ci-dessous pour soumettre l'événement."
#: agenda_culturel/views.py:292
#: agenda_culturel/views.py:315
msgid "This URL has already been submitted, and you can find the event below."
msgstr ""
"Cette URL a déjà été soumise, et vous trouverez l'événement ci-dessous."
#: agenda_culturel/views.py:296
#: agenda_culturel/views.py:319
msgid "This URL has already been submitted and is awaiting moderation."
msgstr "Cette URL a déjà été soumise, et est en attente de modération"
#: agenda_culturel/views.py:321
msgid ""
"This URL has already been submitted, but has not been selected for "
"publication by the moderation team."
@ -387,34 +446,87 @@ msgstr ""
"Cette URL a déjà été soumise, mais n'a pas été retenue par l'équipe de "
"modération pour la publication."
#: agenda_culturel/views.py:298
msgid "This URL has already been submitted and is awaiting moderation."
msgstr "Cette URL a déjà été soumise, et est en attente de modération"
#: agenda_culturel/views.py:320
#: agenda_culturel/views.py:343
msgid "Your message has been sent successfully."
msgstr "L'événement a été supprimé avec succès"
#: agenda_culturel/views.py:328
#: agenda_culturel/views.py:351
msgid "The contact message properties has been successfully modified."
msgstr "Les propriétés du message de contact ont été modifié avec succès."
#: agenda_culturel/views.py:343
#: agenda_culturel/views.py:366
msgid "Open"
msgstr "Ouvert"
#: agenda_culturel/views.py:383
#: agenda_culturel/views.py:406
msgid "Search"
msgstr "Rechercher"
#: agenda_culturel/views.py:491
#: agenda_culturel/views.py:521
msgid "The import has been run successfully."
msgstr "L'import a été lancé avec succès"
#: agenda_culturel/views.py:521
#: agenda_culturel/views.py:537
msgid "The import has been canceled."
msgstr "L'import a été annulé"
#: agenda_culturel/views.py:571
msgid "The recurrent import has been successfully modified."
msgstr "L'import récurrent a été modifié avec succès."
#: agenda_culturel/views.py:577
#, fuzzy
#| msgid "The event has been successfully deleted."
msgid "The recurrent import has been successfully deleted."
msgstr "L'import récurrent a été supprimé avec succès"
#: agenda_culturel/views.py:606
#, fuzzy
#| msgid "The import has been canceled."
msgid "The import has been launched."
msgstr "L'import a été annulé"
#: agenda_culturel/views.py:665
msgid "The merge has been successfully completed."
msgstr "La fusion a été réalisée avec succès."
#: agenda_culturel/views.py:694
msgid "Events have been marked as unduplicated."
msgstr "Les événements ont été marqués comme non dupliqués."
#: agenda_culturel/views.py:711
msgid ""
"The selected event has been retained, while the other has been moved "
"to the recycle bin."
msgstr ""
"L'événement sélectionné a été conservé, l'autre a été déplacé dans la "
"corbeille."
#: agenda_culturel/views.py:713
msgid ""
"The selected event has been retained, while the others have been moved "
"to the recycle bin."
msgstr ""
"L'événement sélectionné a été conservé, les autres ont été déplacés dans la "
"corbeille."
#: agenda_culturel/views.py:719
msgid "The event has been withdrawn from the group and made independent."
msgstr "L'événement a été retiré du groupe et rendu indépendant."
#: agenda_culturel/views.py:765
msgid "The event was successfully duplicated."
msgstr "L'événement a été marqué dupliqué avec succès."
#: agenda_culturel/views.py:768
msgid ""
"The event has been successfully flagged as a duplicate. The moderation team "
"will deal with your suggestion shortly."
msgstr ""
"L'événement a été signalé comme dupliqué avec succès. Votre suggestion sera "
"prochainement prise en charge par l'équipe de modération."
msgid "Add another"
msgstr "Ajouter un autre"

33
src/agenda_culturel/migrations/0028_recurrentimport_batchimportation_recurrentimport.py Normal file
View File

@ -0,0 +1,33 @@
# Generated by Django 4.2.7 on 2024-01-20 15:59
from django.db import migrations, models
import django.db.models.deletion
import django_better_admin_arrayfield.models.fields
class Migration(migrations.Migration):
dependencies = [
('agenda_culturel', '0027_set_dtstart_dtend'),
]
operations = [
migrations.CreateModel(
name='RecurrentImport',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('processor', models.CharField(choices=[('ical', 'ical')], default='ical', max_length=20, verbose_name='Processor')),
('recurrence', models.CharField(choices=[('daily', 'daily'), ('weekly', 'weekly')], default='daily', max_length=10, verbose_name='Import recurrence')),
('source', models.URLField(help_text='URL of the source document', max_length=1024, verbose_name='Source')),
('browsable_url', models.URLField(blank=True, help_text='URL of the corresponding document that will be shown to visitors.', max_length=1024, null=True, verbose_name='Browsable url')),
('defaultLocation', models.CharField(blank=True, help_text='Address for each imported event', max_length=512, null=True, verbose_name='Location')),
('defaultTags', django_better_admin_arrayfield.models.fields.ArrayField(base_field=models.CharField(max_length=64), blank=True, help_text='A list of tags that describe each imported event.', null=True, size=None, verbose_name='Tags for each imported event')),
('defaultCategory', models.ForeignKey(blank=True, help_text='Category of each imported event', null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.category', verbose_name='Category')),
],
),
migrations.AddField(
model_name='batchimportation',
name='recurrentImport',
field=models.ForeignKey(blank=True, editable=False, help_text='Reference to the recurrent import processing', null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.recurrentimport', verbose_name='Recurrent import'),
),
]

21
src/agenda_culturel/migrations/0029_remove_batchimportation_browsable_url_and_more.py Normal file
View File

@ -0,0 +1,21 @@
# Generated by Django 4.2.7 on 2024-01-25 21:37
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('agenda_culturel', '0028_recurrentimport_batchimportation_recurrentimport'),
]
operations = [
migrations.RemoveField(
model_name='batchimportation',
name='browsable_url',
),
migrations.RemoveField(
model_name='batchimportation',
name='source',
),
]

18
src/agenda_culturel/migrations/0030_recurrentimport_downloader.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.2.7 on 2024-01-26 10:15
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('agenda_culturel', '0029_remove_batchimportation_browsable_url_and_more'),
]
operations = [
migrations.AddField(
model_name='recurrentimport',
name='downloader',
field=models.CharField(choices=[('simple', 'simple'), ('chromium headless', 'Headless Chromium')], default='simple', max_length=20, verbose_name='Downloader'),
),
]

45
src/agenda_culturel/models.py
View File

@ -503,8 +503,9 @@ class Event(models.Model):
for e in events:
e.possibly_duplicated = group
# finally update all events (including current)
Event.objects.bulk_update(events + [self], fields=["possibly_duplicated"])
# finally update all events (including current if already created)
elist = list(events) + ([self] if self.pk is not None else [])
Event.objects.bulk_update(elist, fields=["possibly_duplicated"])
def data_fields(all=False):
@ -631,7 +632,7 @@ class Event(models.Model):
nb_updated = Event.objects.bulk_update(to_update, fields = Event.data_fields() + ["imported_date", "modified_date", "uuids", "status"])
nb_draft = 0
if remove_missing_from_source is not None:
if remove_missing_from_source is not None and max_date is not None:
# events that are missing from the import but in database are turned into drafts
# only if they are in the future
@ -704,6 +705,41 @@ class ContactMessage(models.Model):
return ContactMessage.objects.filter(closed=False).count()
class RecurrentImport(models.Model):
class PROCESSOR(models.TextChoices):
ICAL = "ical", _("ical")
class DOWNLOADER(models.TextChoices):
SIMPLE = "simple", _("simple")
CHROMIUMHEADLESS = "chromium headless", _("Headless Chromium")
class RECURRENCE(models.TextChoices):
DAILY = "daily", _("daily"),
WEEKLY = "weekly", _("weekly")
processor = models.CharField(_("Processor"), max_length=20, choices=PROCESSOR.choices, default=PROCESSOR.ICAL)
downloader = models.CharField(_("Downloader"), max_length=20, choices=DOWNLOADER.choices, default=DOWNLOADER.SIMPLE)
recurrence = models.CharField(_("Import recurrence"), max_length=10, choices=RECURRENCE.choices, default=RECURRENCE.DAILY)
source = models.URLField(verbose_name=_('Source'), help_text=_("URL of the source document"), max_length=1024)
browsable_url = models.URLField(verbose_name=_('Browsable url'), help_text=_("URL of the corresponding document that will be shown to visitors."), max_length=1024, blank=True, null=True)
defaultLocation = models.CharField(verbose_name=_('Location'), help_text=_('Address for each imported event'), max_length=512, null=True, blank=True)
defaultCategory = models.ForeignKey(Category, verbose_name=_('Category'), help_text=_('Category of each imported event'), blank=True, null=True, on_delete=models.SET_NULL)
defaultTags = ArrayField(models.CharField(max_length=64), verbose_name=_('Tags for each imported event'), help_text=_("A list of tags that describe each imported event."), blank=True, null=True)
def nb_imports(self):
return BatchImportation.objects.filter(recurrentImport=self).count()
def get_absolute_url(self):
return reverse("view_rimport", kwargs={"pk": self.pk})
class BatchImportation(models.Model):
class STATUS(models.TextChoices):
@ -715,8 +751,7 @@ class BatchImportation(models.Model):
created_date = models.DateTimeField(auto_now_add=True)
source = models.URLField(verbose_name=_('Source'), help_text=_("URL of the source document"), max_length=1024, blank=True, null=True)
browsable_url = models.URLField(verbose_name=_('Browsable url'), help_text=_("URL of the corresponding document that will be shown to visitors."), max_length=1024, blank=True, null=True)
recurrentImport = models.ForeignKey(RecurrentImport, verbose_name=_('Recurrent import'), help_text=_('Reference to the recurrent import processing'), blank=True, null=True, on_delete=models.SET_NULL, editable=False)
status = models.CharField(_("Status"), max_length=20, choices=STATUS.choices, default=STATUS.RUNNING)

6
src/agenda_culturel/static/style.scss
View File

@ -774,4 +774,10 @@ article>article {
.strike {
text-decoration: line-through;
}
.buttons {
display: flex;
flex: auto;
gap: 0.4em;
}

48
src/agenda_culturel/templates/agenda_culturel/batch-imports-inc.html Normal file
View File

@ -0,0 +1,48 @@
<table role="grid">
<thead>
<tr>
<th rowspan="2">Identifiant</th>
<th rowspan="2">Date</th>
<th rowspan="2">Status</th>
<th rowspan="2">Action</th>
<th colspan="4">événements</th>
</tr>
<tr>
<th>initial</th>
<th>importés</th>
<th>mis à jour</th>
<th>dépubliés</th>
</tr>
</thead>
<tbody>
{% for obj in paginator_filter %}
<tr>
<td>{{ obj.id }}</a></td>
<td>{{ obj.created_date }}</td>
<td><span{% if obj.status == "failed" %} data-tooltip="{{ obj.error_message }}"{% endif %}>{{ obj.status }}</span></td>
<td>{% if obj.status == "running" %}<a href="{% url 'cancel_import' obj.id %}">Annuler</a>{% endif %}</td>
<td>{% if obj.status == "success" %}{{ obj.nb_initial }}{% endif %}</td>
<td>{% if obj.status == "success" %}{{ obj.nb_imported }}{% endif %}</td>
<td>{% if obj.status == "success" %}{{ obj.nb_updated }}{% endif %}</td>
<td>{% if obj.status == "success" %}{{ obj.nb_removed }}{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
<footer>
<span>
{% if paginator_filter.has_previous %}
<a href="?page=1" role="button">&laquo; premier</a>
<a href="?page={{ paginator_filter.previous_page_number }}" role="button">précédent</a>
{% endif %}
<span>
Page {{ paginator_filter.number }} sur {{ paginator_filter.paginator.num_pages }}
</span>
{% if paginator_filter.has_next %}
<a href="?page={{ paginator_filter.next_page_number }}" role="button">suivant</a>
<a href="?page={{ paginator_filter.paginator.num_pages }}" role="button">dernier &raquo;</a>
{% endif %}
</span>
</footer>

4
src/agenda_culturel/templates/agenda_culturel/batchimportation_form.html
View File

@ -1,12 +1,12 @@
{% extends "agenda_culturel/page.html" %}
{% load static %}
{% block title %}Importation par lot{% endblock %}
{% block title %}Importation manuelle{% endblock %}
{% block content %}
<h1>Importation par lot</h1>
<h1>Importation manuelle</h1>
<article>
<form method="post">{% csrf_token %}

51
src/agenda_culturel/templates/agenda_culturel/imports.html
View File

@ -12,58 +12,11 @@
<div class="grid two-columns">
<article>
<header>
<a class="slide-buttons" href="{% url 'add_import'%}" role="button">Nouvel import</a>
<a class="slide-buttons" href="{% url 'add_import'%}" role="button">Import manuel</a>
<h1>Importations par lot</h1>
</header>
<table role="grid">
<thead>
<tr>
<th rowspan="2">Identifiant</th>
<th rowspan="2">Date</th>
<th rowspan="2">Status</th>
<th rowspan="2">Action</th>
<th colspan="4">événements</th>
</tr>
<tr>
<th>initial</th>
<th>importés</th>
<th>mis à jour</th>
<th>dépubliés</th>
</tr>
</thead>
<tbody>
{% for obj in paginator_filter %}
<tr>
<td>{{ obj.id }}</a></td>
<td>{{ obj.created_date }}</td>
<td><span{% if obj.status == "failed" %} data-tooltip="{{ obj.error_message }}"{% endif %}>{{ obj.status }}</span></td>
<td>{% if obj.status == "running" %}<a href="{% url 'cancel_import' obj.id %}">Annuler</a>{% endif %}</td>
<td>{% if obj.status == "success" %}{{ obj.nb_initial }}{% endif %}</td>
<td>{% if obj.status == "success" %}{{ obj.nb_imported }}{% endif %}</td>
<td>{% if obj.status == "success" %}{{ obj.nb_updated }}{% endif %}</td>
<td>{% if obj.status == "success" %}{{ obj.nb_removed }}{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
<footer>
<span>
{% if paginator_filter.has_previous %}
<a href="?page=1" role="button">&laquo; premier</a>
<a href="?page={{ paginator_filter.previous_page_number }}" role="button">précédent</a>
{% endif %}
<span>
Page {{ paginator_filter.number }} sur {{ paginator_filter.paginator.num_pages }}
</span>
{% if paginator_filter.has_next %}
<a href="?page={{ paginator_filter.next_page_number }}" role="button">suivant</a>
<a href="?page={{ paginator_filter.paginator.num_pages }}" role="button">dernier &raquo;</a>
{% endif %}
</span>
</footer>
{% include "agenda_culturel/batch-imports-inc.html" with paginator_filter=paginator_filter %}
</article>
{% include "agenda_culturel/side-nav.html" with current="imports" %}

53
src/agenda_culturel/templates/agenda_culturel/page-rimport.html Normal file
View File

@ -0,0 +1,53 @@
{% extends "agenda_culturel/page.html" %}
{% block title %}Import récurrent #{{ object.pk }}{% endblock %}
{% load cat_extra %}
{% load utils_extra %}
{% block entete_header %}
{% css_categories %}
{% endblock %}
{% block content %}
<div class="grid two-columns">
<article>
<header>
<a href="{% url 'recurrent_imports' %}" role="button">&lt; Retour</a>
<div class="buttons slide-buttons">
<a href="{% url 'run_rimport' object.pk %}" role="button">Exécuter {% picto_from_name "download-cloud" %}</a>
<a href="{% url 'edit_rimport' object.pk %}" role="button">Éditer {% picto_from_name "edit" %}</a>
<a href="{% url 'delete_rimport' object.pk %}" role="button">Supprimer {% picto_from_name "trash" %}</a>
</div>
<h1>Import récurrent #{{ object.pk }}</h1>
<ul>
<li><strong>Processeur&nbsp;:</strong> {{ object.processor }}</li>
<li><strong>Téléchargeur&nbsp;:</strong> {{ object.downloader }}</li>
<li><strong>Recurrence&nbsp;:</strong> {{ object.recurrence }}</li>
<li><strong>Source&nbsp;:</strong> <a href="{{ object.source }}">{{ object.source }}</a></li>
<li><strong>Adresse naviguable&nbsp;:</strong> <a href="{{ object.browsable_url }}">{{ object.browsable_url }}</a></li>
<li><strong>Valeurs par défaut&nbsp;:</strong>
<ul>
<li><strong>Localisation&nbsp;:</strong> {{ object.defaultLocation }}</li>
<li><strong>Catégorie&nbsp;:</strong> {{ object.defaultCategory }}</li>
<li><strong>Étiquettes&nbsp;:</strong>
{% for tag in object.defaultTags %}
{{ tag }}{% if not forloop.last %}, {% endif %}
{% endfor %}
</li>
</ul>
</li>
</ul>
</header>
{% include "agenda_culturel/batch-imports-inc.html" with paginator_filter=paginator_filter %}
</article>
{% include "agenda_culturel/side-nav.html" with current="rimports" %}
</div>
{% endblock %}

18
src/agenda_culturel/templates/agenda_culturel/recurrentimport_confirm_delete.html Normal file
View File

@ -0,0 +1,18 @@
{% extends "agenda_culturel/page.html" %}
{% block title %}Supprimer {{ object.title }}{% endblock %}
{% block content %}
<h1>Suppression de l'import récurrent {{ object.pk }}</h1>
<form method="post">{% csrf_token %}
<p>Êtes-vous sûr·e de vouloir supprimer l'import récurrent «&nbsp;{{ object.pk }}&nbsp;»&nbsp; correspondant à la source <a href="{{ object.source }}">{{ object.source }}</a>&nbsp;?</p>
{{ form }}
<div class="grid">
<a href="{% if request.META.HTTP_REFERER %}{{ request.META.HTTP_REFERER }}{% else %}{{ object.get_absolute_url }}{% endif %}" role="button" class="secondary">Annuler</a>
<input type="submit" value="Confirmer">
</div>
</form>
{% endblock %}

27
src/agenda_culturel/templates/agenda_culturel/recurrentimport_form.html Normal file
View File

@ -0,0 +1,27 @@
{% extends "agenda_culturel/page.html" %}
{% load static %}
{% block title %}Importation récurrente{% endblock %}
{% block entete_header %}
<script src="/static/admin/js/vendor/jquery/jquery.js"></script>
<script src="/static/admin/js/jquery.init.js"></script>
<link href="{% static 'css/django_better_admin_arrayfield.min.css' %}" type="text/css" media="all" rel="stylesheet">
<script src="{% static 'js/django_better_admin_arrayfield.min.js' %}"></script>
{% endblock %}
{% block content %}
<h1>Importation récurrente</h1>
<article>
<form method="post">{% csrf_token %}
{{ form.as_p }}
<div class="grid">
<a href="{% if request.META.HTTP_REFERER %}{{ request.META.HTTP_REFERER }}{% else %}{{ object.get_absolute_url }}{% endif %}" role="button" class="secondary">Annuler</a>
<input type="submit" value="Envoyer">
</div>
</form>
</article>
{% endblock %}

75
src/agenda_culturel/templates/agenda_culturel/rimports.html Normal file
View File

@ -0,0 +1,75 @@
{% extends "agenda_culturel/page.html" %}
{% block title %}Importations récurrentes{% endblock %}
{% load utils_extra %}
{% load cat_extra %}
{% block entete_header %}
{% css_categories %}
{% endblock %}
{% block content %}
<div class="grid two-columns">
<article>
<header>
<a class="slide-buttons" href="{% url 'add_rimport'%}" role="button">Ajouter {% picto_from_name "plus-circle" %}</a>
<h1>Importations récurrentes</h1>
</header>
<table role="grid">
<thead>
<tr>
<th>Identifiant</th>
<th>Processeur</th>
<th>Téléchargeur</th>
<th>Récurrence</th>
<th>Source</th>
<th>Adresse naviguable</th>
<th>Nombre d'imports</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{% for obj in paginator_filter %}
<tr>
<td>{{ obj.pk }}</a></td>
<td>{{ obj.processor }}</td>
<td>{{ obj.downloader }}</td>
<td>{{ obj.recurrence }}</td>
<td><a href="{{ obj.source }}">{{ obj.source|truncatechars_middle:32 }}</a></td>
<td><a href="{{ obj.browsable_url }}">{{ obj.browsable_url|truncatechars_middle:32 }}</a></td>
<td>{{ obj.nb_imports }}</td>
<td>
<div class="buttons">
<a href="{% url 'run_rimport' obj.pk %}" role="button">Exécuter</a>
<a href="{% url 'view_rimport' obj.pk %}" role="button">Consulter</a>
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<footer>
<span>
{% if paginator_filter.has_previous %}
<a href="?page=1" role="button">&laquo; premier</a>
<a href="?page={{ paginator_filter.previous_page_number }}" role="button">précédent</a>
{% endif %}
<span>
Page {{ paginator_filter.number }} sur {{ paginator_filter.paginator.num_pages }}
</span>
{% if paginator_filter.has_next %}
<a href="?page={{ paginator_filter.next_page_number }}" role="button">suivant</a>
<a href="?page={{ paginator_filter.paginator.num_pages }}" role="button">dernier &raquo;</a>
{% endif %}
</span>
</footer>
</article>
{% include "agenda_culturel/side-nav.html" with current="rimports" %}
</div>
{% endblock %}

25
src/agenda_culturel/templates/agenda_culturel/run_rimport_confirm.html Normal file
View File

@ -0,0 +1,25 @@
{% extends "agenda_culturel/page.html" %}
{% block title %}Lancer l'import #{{ object.pk }}{% endblock %}
{% block content %}
<article>
<header>
<h1>Lancement de l'import #{{ object.pk }}</h1>
</header>
<form method="post">{% csrf_token %}
<p>Êtes-vous sûr·e de vouloir lancer l'import récurrent «&nbsp;{{ object.pk }}&nbsp;»&nbsp; correspondant à la source <a href="{{ object.source }}">{{ object.source }}</a>&nbsp;?</p>
{{ form }}
<footer>
<div class="grid">
<a href="{% if request.META.HTTP_REFERER %}{{ request.META.HTTP_REFERER }}{% else %}{% url 'recurrent_imports' %}{% endif %}" role="button" class="secondary">Annuler</a>
<input type="submit" value="Confirmer">
</div>
</footer>
</form>
</article>
{% endblock %}

12
src/agenda_culturel/templates/agenda_culturel/side-nav.html
View File

@ -21,7 +21,17 @@
<li><a {% if current == "moderation" %}class="selected" {% endif %}href="{% url 'moderation' %}">Derniers événements soumis</a>{% show_badges_events "left" %}</li>
<li><a {% if current == "contactmessages" %}class="selected" {% endif %}href="{% url 'contactmessages' %}">Messages de contact</a>{% show_badge_contactmessages "left" %}</li>
<li><a {% if current == "duplicates" %}class="selected" {% endif %}href="{% url 'duplicates' %}">Gestion des doublons</a>{% show_badge_duplicated "left" %}</li>
<li><a {% if current == "imports" %}class="selected" {% endif %}href="{% url 'imports' %}">Importations par lot</a></li>
</ul>
</nav>
</article>
<article>
<header>
<h2>Importations</h2>
</header>
<nav>
<ul>
<li><a {% if current == "imports" %}class="selected" {% endif %}href="{% url 'imports' %}">Historiques des importations</a></li>
<li><a {% if current == "rimports" %}class="selected" {% endif %}href="{% url 'recurrent_imports' %}">Importations récurrentes</a></li>
</ul>
</nav>
</article>

13
src/agenda_culturel/templatetags/utils_extra.py
View File

@ -1,5 +1,6 @@
from django import template
from django.utils.safestring import mark_safe
from django.template.defaultfilters import stringfilter
from urllib.parse import urlparse
from datetime import timedelta, date
@ -71,3 +72,15 @@ def picto_from_name(name, datatooltip=""):
@register.filter
def int_to_abc(d):
return auc[int(d)]
@register.filter(is_safe=True)
@stringfilter
def truncatechars_middle(value, arg):
try:
ln = int(arg)
except ValueError:
return value
if len(value) <= ln:
return value
else:
return '{}...{}'.format(value[:ln//2], value[-((ln+1)//2):])

8
src/agenda_culturel/urls.py
View File

@ -38,8 +38,14 @@ urlpatterns = [
path('contactmessages', contactmessages, name='contactmessages'),
path('contactmessage/<int:pk>', ContactMessageUpdateView.as_view(), name='contactmessage'),
path("imports/", imports, name="imports"),
path("imports/add", BatchImportationCreateView.as_view(), name="add_import"),
path("imports/add", add_import, name="add_import"),
path("imports/<int:pk>/cancel", cancel_import, name="cancel_import"),
path("rimports/", recurrent_imports, name="recurrent_imports"),
path("rimports/add", RecurrentImportCreateView.as_view(), name="add_rimport"),
path("rimports/<int:pk>/view", view_rimport, name="view_rimport"),
path("rimports/<int:pk>/edit", RecurrentImportUpdateView.as_view(), name="edit_rimport"),
path("rimports/<int:pk>/delete", RecurrentImportDeleteView.as_view(), name="delete_rimport"),
path("rimports/<int:pk>/run", run_rimport, name="run_rimport"),
path("duplicates/", duplicates, name="duplicates"),
path("duplicates/<int:pk>", DuplicatedEventsDetailView.as_view(), name="view_duplicate"),
path("duplicates/<int:pk>/fix", fix_duplicate, name="fix_duplicate"),

117
src/agenda_culturel/views.py
View File

@ -11,9 +11,9 @@ from django.http import HttpResponseRedirect
from django.urls import reverse
import urllib
from .forms import EventSubmissionForm, EventForm, BatchImportationForm, FixDuplicates, SelectEventInList, MergeDuplicates
from .forms import EventSubmissionForm, EventForm, BatchImportationForm, FixDuplicates, SelectEventInList, MergeDuplicates, RecurrentImportForm
from .models import Event, Category, StaticContent, ContactMessage, BatchImportation, DuplicatedEvents
from .models import Event, Category, StaticContent, ContactMessage, BatchImportation, DuplicatedEvents, RecurrentImport
from django.utils import timezone
from enum import StrEnum
from datetime import date, timedelta
@ -32,7 +32,7 @@ from django.contrib.messages.views import SuccessMessageMixin
from .calendar import CalendarMonth, CalendarWeek, CalendarDay
from .extractors import ExtractorAllURLs
from .celery import app as celery_app, import_events_from_json, import_events_from_url
from .celery import app as celery_app, import_events_from_json, run_recurrent_import
import unicodedata
import logging
@ -507,28 +507,21 @@ def imports(request):
return render(request, 'agenda_culturel/imports.html', {'paginator_filter': response} )
class BatchImportationCreateView(SuccessMessageMixin, LoginRequiredMixin, CreateView):
model = BatchImportation
@login_required(login_url="/accounts/login/")
def add_import(request):
form = BatchImportationForm()
success_url = reverse_lazy('imports')
success_message = _('The import has been run successfully.')
if request.method == 'POST':
form = BatchImportationForm(request.POST)
form_class = BatchImportationForm
if form.is_valid():
def form_valid(self, form):
# run import
if "json" in form.data and form.data["json"] is not None and form.data["json"].strip() != "":
result = import_events_from_json.delay(form.data["json"])
else:
result = import_events_from_url.delay(form.data["source"], form.data["browsable_url"])
# update the object with celery_id
form.instance.celery_id = result.id
return super().form_valid(form)
messages.success(request, _("The import has been run successfully."))
return HttpResponseRedirect(reverse_lazy("imports"))
return render(request, 'agenda_culturel/batchimportation_form.html', {"form": form})
@login_required(login_url="/accounts/login/")
@ -547,6 +540,78 @@ def cancel_import(request, pk):
cancel_url = reverse_lazy("imports")
return render(request, 'agenda_culturel/cancel_import_confirm.html', {"object": import_process, "cancel_url": cancel_url})
#########################
## recurrent importations
#########################
@login_required(login_url="/accounts/login/")
def recurrent_imports(request):
paginator = Paginator(RecurrentImport.objects.all().order_by("-pk"), 10)
page = request.GET.get('page')
try:
response = paginator.page(page)
except PageNotAnInteger:
response = paginator.page(1)
except EmptyPage:
response = paginator.page(paginator.num_pages)
return render(request, 'agenda_culturel/rimports.html', {'paginator_filter': response} )
class RecurrentImportCreateView(LoginRequiredMixin, CreateView):
model = RecurrentImport
success_url = reverse_lazy('recurrent_imports')
form_class = RecurrentImportForm
class RecurrentImportUpdateView(SuccessMessageMixin, LoginRequiredMixin, UpdateView):
model = RecurrentImport
form_class = RecurrentImportForm
success_message = _('The recurrent import has been successfully modified.')
class RecurrentImportDeleteView(SuccessMessageMixin, LoginRequiredMixin, DeleteView):
model = RecurrentImport
success_url = reverse_lazy('recurrent_imports')
success_message = _('The recurrent import has been successfully deleted.')
@login_required(login_url="/accounts/login/")
def view_rimport(request, pk):
obj = get_object_or_404(RecurrentImport, pk=pk)
paginator = Paginator(BatchImportation.objects.filter(recurrentImport=pk).order_by("-created_date"), 10)
page = request.GET.get('page')
try:
response = paginator.page(page)
except PageNotAnInteger:
response = paginator.page(1)
except EmptyPage:
response = paginator.page(paginator.num_pages)
return render(request, 'agenda_culturel/page-rimport.html', {'paginator_filter': response, 'object': obj} )
@login_required(login_url="/accounts/login/")
def run_rimport(request, pk):
rimport = get_object_or_404(RecurrentImport, pk=pk)
if request.method == 'POST':
# run recurrent import
result = run_recurrent_import.delay(pk)
messages.success(request, _("The import has been launched."))
return HttpResponseRedirect(reverse_lazy("view_rimport", args=[pk]))
else:
return render(request, 'agenda_culturel/run_rimport_confirm.html', {"object": rimport })
#########################
## duplicated events
#########################
class DuplicatedEventsDetailView(LoginRequiredMixin, DetailView):
model = DuplicatedEvents
@ -597,7 +662,7 @@ def merge_duplicate(request, pk):
e.status = Event.STATUS.TRASH
Event.objects.bulk_update(events, fields=["status"])
messages.info(request, _("La fusion a été réalisée avec succès."))
messages.info(request, _("The merge has been successfully completed."))
return HttpResponseRedirect(new_event.get_absolute_url())
return render(request, 'agenda_culturel/merge_duplicate.html', context={'form': form, 'object': edup})
@ -626,7 +691,7 @@ def fix_duplicate(request, pk):
s_event = events[0]
date = s_event.start_day
messages.success(request, _("Les événements ont été marqués comme non dupliqués."))
messages.success(request, _("Events have been marked as unduplicated."))
edup.delete()
if date is None:
return HttpResponseRedirect(reverse_lazy("home"))
@ -643,15 +708,15 @@ def fix_duplicate(request, pk):
url = selected.get_absolute_url()
edup.delete()
if nb == 1:
messages.success(request, _("L'événement sélectionné a été conservé, l'autre a été déplacé dans la corbeille."))
messages.success(request, _("The selected event has been retained, while the other has been moved to the recycle bin."))
else:
messages.success(request, _("L'événement sélectionné a été conservé, les autres ont été déplacés dans la corbeille."))
messages.success(request, _("The selected event has been retained, while the others have been moved to the recycle bin."))
return HttpResponseRedirect(url)
elif form.is_action_remove():
event = form.get_selected_event(edup)
event.possibly_duplicated = None
event.save()
messages.success(request, _("L'événement a été retiré du groupe et rendu indépendant."))
messages.success(request, _("The event has been withdrawn from the group and made independent."))
if edup.nb_duplicated() == 1:
return HttpResponseRedirect(event.get_absolute_url())
else:
@ -697,10 +762,10 @@ def set_duplicate(request, year, month, day, pk):
event.set_possibly_duplicated(selected)
event.save()
if request.user.is_authenticated:
messages.success(request, _("L'événement a été marqué dupliqué avec succès."))
messages.success(request, _("The event was successfully duplicated."))
return HttpResponseRedirect(reverse_lazy("view_duplicate", args=[event.possibly_duplicated.pk]))
else:
messages.info(request, _("L'événement a été signalé comme dupliqué avec succès. Votre suggestion sera prochainement prise en charge par l'équipe de modération."))
messages.info(request, _("The event has been successfully flagged as a duplicate. The moderation team will deal with your suggestion shortly."))
return HttpResponseRedirect(event.get_absolute_url())

3
src/requirements.txt
View File

@ -33,3 +33,6 @@ django-compressor==4.4
django-libsass==0.9
django-ckeditor==6.7.0
django-recurrence==1.11.1
icalendar==5.0.11
lxml==5.1.0