diff --git a/experimentations/get_ical_events.py b/experimentations/get_ical_events.py index 6ef0242..0920f59 100755 --- a/experimentations/get_ical_events.py +++ b/experimentations/get_ical_events.py @@ -1,249 +1,29 @@ #!/usr/bin/python3 # coding: utf-8 - -from abc import ABC, abstractmethod -from urllib.parse import urlparse -import urllib.request import os -from selenium import webdriver -from selenium.webdriver.chrome.service import Service -from selenium.webdriver.chrome.options import Options -import icalendar -from icalendar import vDatetime -from datetime import datetime, date import json -from bs4 import BeautifulSoup -import pickle +import sys +# getting the name of the directory +# where the this file is present. +current = os.path.dirname(os.path.realpath(__file__)) + +# Getting the parent directory name +# where the current directory is present. +parent = os.path.dirname(current) + +# adding the parent directory to +# the sys.path. +sys.path.append(parent) -class Downloader(ABC): - - def __init__(self): - pass - - @abstractmethod - def download(self, url): - pass - -class SimpleDownloader(Downloader): - - def __init__(self): - super().__init__() - - - def download(self, url): - print("Downloading {}".format(url)) - - try: - resource = urllib.request.urlopen(url) - data = resource.read().decode(resource.headers.get_content_charset()) - return data - except: - return None +from src.agenda_culturel.import_tasks.downloader import * +from src.agenda_culturel.import_tasks.extractor import * +from src.agenda_culturel.import_tasks.importer import * +from src.agenda_culturel.import_tasks.extractor_ical import * -class ChromiumHeadlessDownloader(Downloader): - - def __init__(self): - super().__init__() - options = Options() - options.add_argument("--headless=new") - service = Service("/usr/bin/chromedriver") - self.driver = webdriver.Chrome(service=service, options=options) - - - def download(self, url): - print("Download {}".format(url)) - - self.driver.get(url) - return driver.page_source - - -class Extractor(ABC): - - def __init__(self): - self.header = {} - self.events = [] - - @abstractmethod - def extract(self, content, url, url_human = None): - pass - - def set_header(self, url): - self.header["url"] = url - self.header["date"] = datetime.now() - - def clear_events(self): - self.events = [] - - def add_event(self, title, category, start_day, location, description, tags, uuid, recurrences=None, url_human=None, start_time=None, end_day=None, end_time=None, last_modified=None, published=False): - if title is None: - print("ERROR: cannot import an event without name") - return - if start_day is None: - print("ERROR: cannot import an event without start day") - return - - event = { - "title": title, - "category": category, - "start_day": start_day, - "uuid": uuid, - "location": location, - "description": description, - "tags": tags, - "published": published - } - if url_human is not None: - event["url_human"] = url_human - if start_time is not None: - event["start_time"] = start_time - if end_day is not None: - event["end_day"] = end_day - if end_time is not None: - event["end_time"] = end_time - - if last_modified is not None: - event["last_modified"] = last_modified - - if recurrences is not None: - event["recurrences"] = recurrences - - self.events.append(event) - - def default_value_if_exists(self, default_values, key): - return default_values[key] if default_values is not None and key in default_values else None - - def get_structure(self): - return { "header": self.header, "events": self.events} - -class ICALExtractor(Extractor): - - def __init__(self): - super().__init__() - - def get_item_from_vevent(self, event, name, raw = False): - try: - r = event.decoded(name) - if raw: - return r - else: - return r.decode() - except: - return None - - def get_dt_item_from_vevent(self, event, name): - item = self.get_item_from_vevent(event, name, raw = True) - - day = None - time = None - - if item is not None: - if isinstance(item, datetime): - day = item.date() - time = item.time() - elif isinstance(item, date): - day = item - time = None - - return day, time - - - def extract(self, content, url, url_human = None, default_values = None, published = False): - print("Extracting ical events from {}".format(url)) - self.set_header(url) - self.clear_events() - self.uuids = {} - - calendar = icalendar.Calendar.from_ical(content) - - for event in calendar.walk('VEVENT'): - title = self.get_item_from_vevent(event, "SUMMARY") - category = self.default_value_if_exists(default_values, "category") - - start_day, start_time = self.get_dt_item_from_vevent(event, "DTSTART") - - end_day, end_time = self.get_dt_item_from_vevent(event, "DTEND") - - location = self.get_item_from_vevent(event, "LOCATION") - if location is None: - location = self.default_value_if_exists(default_values, "location") - - description = self.get_item_from_vevent(event, "DESCRIPTION") - if description is not None: - soup = BeautifulSoup(description) - delimiter = '\n' - for line_break in soup.findAll('br'): - line_break.replaceWith(delimiter) - description = soup.get_text() - - last_modified = self.get_item_from_vevent(event, "LAST_MODIFIED") - - uuid = self.get_item_from_vevent(event, "UID") - - if uuid is not None: - if uuid in self.uuids: - self.uuids[uuid] += 1 - uuid += ":{:04}".format(self.uuids[uuid] - 1) - else: - self.uuids[uuid] = 1 - event_url = url + "#" + uuid - - tags = self.default_value_if_exists(default_values, "tags") - - last_modified = self.get_item_from_vevent(event, "LAST-MODIFIED", raw = True) - - recurrence_entries = {} - for e in ["RRULE", "EXRULE", "EXDATE", "RDATE"]: - i = self.get_item_from_vevent(event, e, raw = True) - if i is not None: - recurrence_entries[e] = i - - if start_day is not None and len(recurrence_entries) != 0: - recurrences = "" - - for k, r in recurrence_entries.items(): - if isinstance(r, list): - recurrences += "\n".join([k + ":" + e.to_ical().decode() for e in r]) + "\n" - else: - recurrences += k + ":" + r.to_ical().decode() + "\n" - else: - recurrences = None - - - self.add_event(title, category, start_day, location, description, tags, recurrences=recurrences, uuid=event_url, url_human=url_human, start_time=start_time, end_day=end_day, end_time=end_time, last_modified=last_modified, published=published) - - return self.get_structure() - - - -class URL2Events: - - def __init__(self, downloader, extractor): - - self.downloader = downloader - self.extractor = extractor - - def process(self, url, url_human = None, cache = None, default_values = None, published = False): - - if cache and os.path.exists(cache): - print("Loading cache ({})".format(cache)) - with open(cache) as f: - content = "\n".join(f.readlines()) - else: - content = self.downloader.download(url) - - if cache: - print("Saving cache ({})".format(cache)) - dir = os.path.dirname(cache) - if dir != "" and not os.path.exists(dir): - os.makedirs(dir) - with open(cache, "w") as text_file: - text_file.write(content) - - return self.extractor.extract(content, url, url_human, default_values, published) if __name__ == "__main__": diff --git a/src/agenda_culturel/admin.py b/src/agenda_culturel/admin.py index fce2f6d..c73b08c 100644 --- a/src/agenda_culturel/admin.py +++ b/src/agenda_culturel/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin from django import forms -from .models import Event, Category, StaticContent, DuplicatedEvents, BatchImportation +from .models import Event, Category, StaticContent, DuplicatedEvents, BatchImportation, RecurrentImport from django_better_admin_arrayfield.admin.mixins import DynamicArrayMixin from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget from django_better_admin_arrayfield.models.fields import DynamicArrayField @@ -10,6 +10,7 @@ admin.site.register(Category) admin.site.register(StaticContent) admin.site.register(DuplicatedEvents) admin.site.register(BatchImportation) +admin.site.register(RecurrentImport) class URLWidget(DynamicArrayWidget): diff --git a/src/agenda_culturel/celery.py b/src/agenda_culturel/celery.py index eaaa6ca..7b2976c 100644 --- a/src/agenda_culturel/celery.py +++ b/src/agenda_culturel/celery.py @@ -1,4 +1,5 @@ import os +import json from celery import Celery from celery.schedules import crontab @@ -6,6 +7,13 @@ from celery.utils.log import get_task_logger from .extractors import ExtractorAllURLs +from .import_tasks.downloader import * +from .import_tasks.extractor import * +from .import_tasks.importer import * +from .import_tasks.extractor_ical import * + + + # Set the default Django settings module for the 'celery' program. APP_ENV = os.getenv("APP_ENV", "dev") @@ -44,14 +52,20 @@ def close_import_task(taskid, success, error_message, importer): @app.task(bind=True) def import_events_from_json(self, json): - from agenda_culturel.models import Event + from agenda_culturel.models import Event, BatchImportation from .importation import EventsImporter + # create a batch importation + importation = BatchImportation(celery_id=self.request.id) + # save batch importation + importation.save() + + logger.info("Import events from json: {}".format(self.request.id)) importer = EventsImporter(self.request.id) - # try: + #try: success, error_message = importer.import_events(json) # finally, close task @@ -62,25 +76,55 @@ def import_events_from_json(self, json): @app.task(bind=True) -def import_events_from_url(self, source, browsable_url): - from agenda_culturel.models import Event +def run_recurrent_import(self, pk): + from agenda_culturel.models import RecurrentImport, BatchImportation from .importation import EventsImporter + from django.shortcuts import get_object_or_404 - logger.info("Import events from url: {} {}".format(source, self.request.id)) - - # first get json - # TODO + logger.info("Run recurrent import: {}".format(self.request.id)) - # then import - # importer = EventsImporter(self.request.id) - # success, error_message = importer.import_events(json) + # get the recurrent import + rimport = RecurrentImport.objects.get(pk=pk) + # create a batch importation + importation = BatchImportation(recurrentImport=rimport, celery_id=self.request.id) + # save batch importation + importation.save() - success = True - error_message = "" + # create an importer + importer = EventsImporter(self.request.id) - # finally, close task - close_import_task(self.request.id, success, error_message) + # prepare downloading and extracting processes + downloader = SimpleDownloader() if rimport.downloader == RecurrentImport.DOWNLOADER.SIMPLE else ChromiumHeadlessDownloader() + extractor = ICALExtractor() if rimport.processor == RecurrentImport.PROCESSOR.ICAL else None + + if extractor is None: + logger.error("Unknown extractor") + close_import_task(self.request.id, False, "Unknown extractor", importer) + + # set parameters + u2e = URL2Events(downloader, extractor) + url = rimport.source + browsable_url = rimport.browsable_url + category = str(rimport.defaultCategory) + location = rimport.defaultLocation + tags = rimport.defaultTags + + try: + # get events from website + events = u2e.process(url, browsable_url, default_values = {"category": category, "location": location, "tags": tags}, published = True) + + # convert it to json + json_events = json.dumps(events, default=str) + + # import events (from json) + success, error_message = importer.import_events(json_events) + + # finally, close task + close_import_task(self.request.id, success, error_message, importer) + except Exception as e: + logger.error(e) + close_import_task(self.request.id, False, e, importer) diff --git a/src/agenda_culturel/forms.py b/src/agenda_culturel/forms.py index fd0accd..7e600c3 100644 --- a/src/agenda_culturel/forms.py +++ b/src/agenda_culturel/forms.py @@ -2,7 +2,7 @@ from django.forms import ModelForm, ValidationError, TextInput, Form, URLField, from datetime import date from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget -from .models import Event, BatchImportation +from .models import Event, BatchImportation, RecurrentImport from django.utils.translation import gettext_lazy as _ from string import ascii_uppercase as auc from .templatetags.utils_extra import int_to_abc @@ -25,6 +25,11 @@ class DynamicArrayWidgetURLs(DynamicArrayWidget): class DynamicArrayWidgetTags(DynamicArrayWidget): template_name = "agenda_culturel/widgets/widget-tags.html" +class RecurrentImportForm(ModelForm): + class Meta: + model = RecurrentImport + fields = '__all__' + class EventForm(ModelForm): class Meta: @@ -75,26 +80,8 @@ class EventForm(ModelForm): -class BatchImportationForm(ModelForm): - - json = CharField(label="JSON (facultatif)", widget=Textarea(attrs={"rows":"10"}), help_text=_("JSON in the format expected for the import. If the JSON is provided here, we will ignore the URLs given above, and use the information provided by the json without importing any additional events from the URL."), required=False) - - class Meta: - model = BatchImportation - fields = ['source', 'browsable_url'] - - def clean(self): - cleaned_data = super().clean() - json = cleaned_data.get("json") - source = cleaned_data.get("source") - browsable_url = cleaned_data.get("browsable_url") - - if (not json or json == "") and (not source or source == "") and (not browsable_url or browsable_url == ""): - raise ValidationError(_("You need to fill in either the json or the source possibly supplemented by the navigable URL.")) - - # Always return a value to use as the new cleaned data, even if - # this method didn't change it. - return cleaned_data +class BatchImportationForm(Form): + json = CharField(label="JSON", widget=Textarea(attrs={"rows":"10"}), help_text=_("JSON in the format expected for the import."), required=True) class FixDuplicates(Form): diff --git a/src/agenda_culturel/import_tasks/downloader.py b/src/agenda_culturel/import_tasks/downloader.py new file mode 100644 index 0000000..ef39f01 --- /dev/null +++ b/src/agenda_culturel/import_tasks/downloader.py @@ -0,0 +1,50 @@ +from urllib.parse import urlparse +import urllib.request +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.chrome.options import Options +from abc import ABC, abstractmethod + + +class Downloader(ABC): + + def __init__(self): + pass + + @abstractmethod + def download(self, url): + pass + +class SimpleDownloader(Downloader): + + def __init__(self): + super().__init__() + + + def download(self, url): + print("Downloading {}".format(url)) + + try: + resource = urllib.request.urlopen(url) + data = resource.read().decode(resource.headers.get_content_charset()) + return data + except: + return None + + + +class ChromiumHeadlessDownloader(Downloader): + + def __init__(self): + super().__init__() + options = Options() + options.add_argument("--headless=new") + service = Service("/usr/bin/chromedriver") + self.driver = webdriver.Chrome(service=service, options=options) + + + def download(self, url): + print("Download {}".format(url)) + + self.driver.get(url) + return driver.page_source diff --git a/src/agenda_culturel/import_tasks/extractor.py b/src/agenda_culturel/import_tasks/extractor.py new file mode 100644 index 0000000..768b208 --- /dev/null +++ b/src/agenda_culturel/import_tasks/extractor.py @@ -0,0 +1,62 @@ +from abc import ABC, abstractmethod +from bs4 import BeautifulSoup +from datetime import datetime + + +class Extractor(ABC): + + def __init__(self): + self.header = {} + self.events = [] + + @abstractmethod + def extract(self, content, url, url_human = None): + pass + + def set_header(self, url): + self.header["url"] = url + self.header["date"] = datetime.now() + + def clear_events(self): + self.events = [] + + def add_event(self, title, category, start_day, location, description, tags, uuid, recurrences=None, url_human=None, start_time=None, end_day=None, end_time=None, last_modified=None, published=False): + if title is None: + print("ERROR: cannot import an event without name") + return + if start_day is None: + print("ERROR: cannot import an event without start day") + return + + event = { + "title": title, + "category": category, + "start_day": start_day, + "uuid": uuid, + "location": location, + "description": description, + "tags": tags, + "published": published + } + if url_human is not None: + event["url_human"] = url_human + if start_time is not None: + event["start_time"] = start_time + if end_day is not None: + event["end_day"] = end_day + if end_time is not None: + event["end_time"] = end_time + + if last_modified is not None: + event["last_modified"] = last_modified + + if recurrences is not None: + event["recurrences"] = recurrences + + self.events.append(event) + + def default_value_if_exists(self, default_values, key): + return default_values[key] if default_values is not None and key in default_values else None + + def get_structure(self): + return { "header": self.header, "events": self.events} diff --git a/src/agenda_culturel/import_tasks/extractor_ical.py b/src/agenda_culturel/import_tasks/extractor_ical.py new file mode 100644 index 0000000..7d812c3 --- /dev/null +++ b/src/agenda_culturel/import_tasks/extractor_ical.py @@ -0,0 +1,110 @@ +import icalendar +import warnings + +from icalendar import vDatetime +from datetime import datetime, date +from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning + +from .extractor import * + + +class ICALExtractor(Extractor): + + def __init__(self): + super().__init__() + + def get_item_from_vevent(self, event, name, raw = False): + try: + r = event.decoded(name) + if raw: + return r + else: + return r.decode() + except: + return None + + def get_dt_item_from_vevent(self, event, name): + item = self.get_item_from_vevent(event, name, raw = True) + + day = None + time = None + + if item is not None: + if isinstance(item, datetime): + day = item.date() + time = item.time() + elif isinstance(item, date): + day = item + time = None + + return day, time + + + def extract(self, content, url, url_human = None, default_values = None, published = False): + warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning) + + print("Extracting ical events from {}".format(url)) + self.set_header(url) + self.clear_events() + self.uuids = {} + + calendar = icalendar.Calendar.from_ical(content) + + for event in calendar.walk('VEVENT'): + title = self.get_item_from_vevent(event, "SUMMARY") + category = self.default_value_if_exists(default_values, "category") + + start_day, start_time = self.get_dt_item_from_vevent(event, "DTSTART") + + end_day, end_time = self.get_dt_item_from_vevent(event, "DTEND") + + location = self.get_item_from_vevent(event, "LOCATION") + if location is None: + location = self.default_value_if_exists(default_values, "location") + + description = self.get_item_from_vevent(event, "DESCRIPTION") + if description is not None: + soup = BeautifulSoup(description, features="lxml") + delimiter = '\n' + for line_break in soup.findAll('br'): + line_break.replaceWith(delimiter) + description = soup.get_text() + + last_modified = self.get_item_from_vevent(event, "LAST_MODIFIED") + + uuid = self.get_item_from_vevent(event, "UID") + + if uuid is not None: + if uuid in self.uuids: + self.uuids[uuid] += 1 + uuid += ":{:04}".format(self.uuids[uuid] - 1) + else: + self.uuids[uuid] = 1 + event_url = url + "#" + uuid + + tags = self.default_value_if_exists(default_values, "tags") + + last_modified = self.get_item_from_vevent(event, "LAST-MODIFIED", raw = True) + + recurrence_entries = {} + for e in ["RRULE", "EXRULE", "EXDATE", "RDATE"]: + i = self.get_item_from_vevent(event, e, raw = True) + if i is not None: + recurrence_entries[e] = i + + if start_day is not None and len(recurrence_entries) != 0: + recurrences = "" + + for k, r in recurrence_entries.items(): + if isinstance(r, list): + recurrences += "\n".join([k + ":" + e.to_ical().decode() for e in r]) + "\n" + else: + recurrences += k + ":" + r.to_ical().decode() + "\n" + else: + recurrences = None + + if title is not None: + self.add_event(title, category, start_day, location, description, tags, recurrences=recurrences, uuid=event_url, url_human=url_human, start_time=start_time, end_day=end_day, end_time=end_time, last_modified=last_modified, published=published) + + return self.get_structure() + diff --git a/src/agenda_culturel/import_tasks/importer.py b/src/agenda_culturel/import_tasks/importer.py new file mode 100644 index 0000000..44b57a9 --- /dev/null +++ b/src/agenda_culturel/import_tasks/importer.py @@ -0,0 +1,31 @@ +import os + +from .downloader import * +from .extractor import * + + +class URL2Events: + + def __init__(self, downloader, extractor): + + self.downloader = downloader + self.extractor = extractor + + def process(self, url, url_human = None, cache = None, default_values = None, published = False): + + if cache and os.path.exists(cache): + print("Loading cache ({})".format(cache)) + with open(cache) as f: + content = "\n".join(f.readlines()) + else: + content = self.downloader.download(url) + + if cache: + print("Saving cache ({})".format(cache)) + dir = os.path.dirname(cache) + if dir != "" and not os.path.exists(dir): + os.makedirs(dir) + with open(cache, "w") as text_file: + text_file.write(content) + + return self.extractor.extract(content, url, url_human, default_values, published) diff --git a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po index ac3f050..d507a05 100644 --- a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po +++ b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: agenda_culturel\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2023-12-29 15:35+0000\n" +"POT-Creation-Date: 2024-01-26 12:27+0000\n" "PO-Revision-Date: 2023-10-29 14:16+0000\n" "Last-Translator: Jean-Marie Favreau \n" "Language-Team: Jean-Marie Favreau \n" @@ -17,340 +17,395 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -#: agenda_culturel/forms.py:38 +#: agenda_culturel/forms.py:62 msgid "The end date must be after the start date." msgstr "La date de fin doit être après la date de début." -#: agenda_culturel/forms.py:53 +#: agenda_culturel/forms.py:77 msgid "The end time cannot be earlier than the start time." msgstr "L'heure de fin ne peut pas être avant l'heure de début." -#: agenda_culturel/forms.py:61 -msgid "" -"JSON in the format expected for the import. If the JSON is provided here, we " -"will ignore the URLs given above, and use the information provided by the " -"json without importing any additional events from the URL." -msgstr "" -"JSON au format attendu pour l'import. Si le JSON est fourni ici, on ignorera " -"les URL données au dessus, et on utilisera les informations fournies par le " -"json sans réaliser d'importation supplémentaire d'événements depuis l'URL." +#: agenda_culturel/forms.py:84 +msgid "JSON in the format expected for the import." +msgstr "JSON dans le format attendu pour l'import" -#: agenda_culturel/forms.py:74 -msgid "" -"You need to fill in either the json or the source possibly supplemented by " -"the navigable URL." -msgstr "" -"Vous devez renseigner soit le json soit la source éventuellement complétée " -"de l'URL navigable." - -#: agenda_culturel/models.py:26 agenda_culturel/models.py:55 -#: agenda_culturel/models.py:465 +#: agenda_culturel/models.py:32 agenda_culturel/models.py:61 +#: agenda_culturel/models.py:695 msgid "Name" msgstr "Nom" -#: agenda_culturel/models.py:26 agenda_culturel/models.py:55 +#: agenda_culturel/models.py:32 agenda_culturel/models.py:61 msgid "Category name" msgstr "Nom de la catégorie" -#: agenda_culturel/models.py:27 +#: agenda_culturel/models.py:33 msgid "Content" msgstr "Contenu" -#: agenda_culturel/models.py:27 +#: agenda_culturel/models.py:33 msgid "Text as shown to the visitors" msgstr "Text tel que présenté aux visiteureuses" -#: agenda_culturel/models.py:28 +#: agenda_culturel/models.py:34 msgid "URL path" msgstr "" -#: agenda_culturel/models.py:28 +#: agenda_culturel/models.py:34 msgid "URL path where the content is included." msgstr "" -#: agenda_culturel/models.py:56 +#: agenda_culturel/models.py:62 msgid "Alternative Name" msgstr "Nom alternatif" -#: agenda_culturel/models.py:56 +#: agenda_culturel/models.py:62 msgid "Alternative name used with a time period" msgstr "Nom alternatif utilisé avec une période de temps" -#: agenda_culturel/models.py:57 +#: agenda_culturel/models.py:63 msgid "Short name" msgstr "Nom court" -#: agenda_culturel/models.py:57 +#: agenda_culturel/models.py:63 msgid "Short name of the category" msgstr "Nom court de la catégorie" -#: agenda_culturel/models.py:58 +#: agenda_culturel/models.py:64 msgid "Color" msgstr "Couleur" -#: agenda_culturel/models.py:58 +#: agenda_culturel/models.py:64 msgid "Color used as background for the category" msgstr "Couleur utilisée comme fond de la catégorie" -#: agenda_culturel/models.py:95 agenda_culturel/models.py:142 +#: agenda_culturel/models.py:101 agenda_culturel/models.py:168 +#: agenda_culturel/models.py:733 msgid "Category" msgstr "Catégorie" -#: agenda_culturel/models.py:96 +#: agenda_culturel/models.py:102 msgid "Categories" msgstr "Catégories" -#: agenda_culturel/models.py:130 +#: agenda_culturel/models.py:153 msgid "Published" msgstr "Publié" -#: agenda_culturel/models.py:131 +#: agenda_culturel/models.py:154 msgid "Draft" msgstr "Brouillon" -#: agenda_culturel/models.py:132 +#: agenda_culturel/models.py:155 msgid "Trash" msgstr "Corbeille" -#: agenda_culturel/models.py:138 +#: agenda_culturel/models.py:164 msgid "Title" msgstr "Titre" -#: agenda_culturel/models.py:138 +#: agenda_culturel/models.py:164 msgid "Short title" msgstr "Titre court" -#: agenda_culturel/models.py:140 agenda_culturel/models.py:492 +#: agenda_culturel/models.py:166 agenda_culturel/models.py:756 msgid "Status" msgstr "Status" -#: agenda_culturel/models.py:142 +#: agenda_culturel/models.py:168 msgid "Category of the event" msgstr "Catégorie de l'événement" -#: agenda_culturel/models.py:144 +#: agenda_culturel/models.py:170 msgid "Day of the event" msgstr "Date de l'événement" -#: agenda_culturel/models.py:145 +#: agenda_culturel/models.py:171 msgid "Starting time" msgstr "Heure de début" -#: agenda_culturel/models.py:147 +#: agenda_culturel/models.py:173 msgid "End day of the event" msgstr "Fin de l'événement" -#: agenda_culturel/models.py:147 +#: agenda_culturel/models.py:173 msgid "End day of the event, only required if different from the start day." msgstr "" "Date de fin de l'événement, uniquement nécessaire s'il est différent du " "premier jour de l'événement" -#: agenda_culturel/models.py:148 +#: agenda_culturel/models.py:174 msgid "Final time" msgstr "Heure de fin" -#: agenda_culturel/models.py:150 +#: agenda_culturel/models.py:176 +msgid "Recurrence" +msgstr "Récurrence" + +#: agenda_culturel/models.py:178 agenda_culturel/models.py:731 msgid "Location" msgstr "Localisation" -#: agenda_culturel/models.py:150 +#: agenda_culturel/models.py:178 msgid "Address of the event" msgstr "Adresse de l'événement" -#: agenda_culturel/models.py:152 +#: agenda_culturel/models.py:180 msgid "Description" msgstr "Description" -#: agenda_culturel/models.py:152 +#: agenda_culturel/models.py:180 msgid "General description of the event" msgstr "Description générale de l'événement" -#: agenda_culturel/models.py:154 +#: agenda_culturel/models.py:182 msgid "Illustration (local image)" msgstr "Illustration (image locale)" -#: agenda_culturel/models.py:154 +#: agenda_culturel/models.py:182 msgid "Illustration image stored in the agenda server" msgstr "Image d'illustration stockée sur le serveur de l'agenda" -#: agenda_culturel/models.py:156 +#: agenda_culturel/models.py:184 msgid "Illustration" msgstr "Illustration" -#: agenda_culturel/models.py:156 +#: agenda_culturel/models.py:184 msgid "URL of the illustration image" msgstr "URL de l'image illustrative" -#: agenda_culturel/models.py:157 +#: agenda_culturel/models.py:185 msgid "Illustration description" msgstr "Description de l'illustration" -#: agenda_culturel/models.py:157 +#: agenda_culturel/models.py:185 msgid "Alternative text used by screen readers for the image" msgstr "Texte alternatif utiliser par les lecteurs d'écrans pour l'image" -#: agenda_culturel/models.py:159 +#: agenda_culturel/models.py:187 +msgid "Importation source" +msgstr "Source d'importation" + +#: agenda_culturel/models.py:187 +msgid "Importation source used to detect removed entries." +msgstr "Source d'importation utilisée pour détecter les éléments supprimés/" + +#: agenda_culturel/models.py:188 msgid "UUIDs" msgstr "UUIDs" -#: agenda_culturel/models.py:159 +#: agenda_culturel/models.py:188 msgid "UUIDs from import to detect duplicated entries." msgstr "UUIDs utilisés pendant l'import pour détecter les entrées dupliquées" -#: agenda_culturel/models.py:160 +#: agenda_culturel/models.py:189 msgid "URLs" msgstr "URLs" -#: agenda_culturel/models.py:160 +#: agenda_culturel/models.py:189 msgid "List of all the urls where this event can be found." msgstr "Liste de toutes les urls où l'événement peut être trouvé." -#: agenda_culturel/models.py:162 +#: agenda_culturel/models.py:191 msgid "Tags" msgstr "Étiquettes" -#: agenda_culturel/models.py:162 +#: agenda_culturel/models.py:191 msgid "A list of tags that describe the event." msgstr "Une liste d'étiquettes décrivant l'événement" -#: agenda_culturel/models.py:164 +#: agenda_culturel/models.py:193 msgid "Possibly duplicated" msgstr "Possibles doublons" -#: agenda_culturel/models.py:194 +#: agenda_culturel/models.py:234 msgid "Event" msgstr "Événement" -#: agenda_culturel/models.py:195 +#: agenda_culturel/models.py:235 msgid "Events" msgstr "Événements" -#: agenda_culturel/models.py:464 +#: agenda_culturel/models.py:694 msgid "Subject" msgstr "Sujet" -#: agenda_culturel/models.py:464 +#: agenda_culturel/models.py:694 msgid "The subject of your message" msgstr "Sujet de votre message" -#: agenda_culturel/models.py:465 +#: agenda_culturel/models.py:695 msgid "Your name" msgstr "Votre nom" -#: agenda_culturel/models.py:466 +#: agenda_culturel/models.py:696 msgid "Email address" msgstr "Adresse email" -#: agenda_culturel/models.py:466 +#: agenda_culturel/models.py:696 msgid "Your email address" msgstr "Votre adresse email" -#: agenda_culturel/models.py:467 +#: agenda_culturel/models.py:697 msgid "Message" msgstr "Message" -#: agenda_culturel/models.py:467 +#: agenda_culturel/models.py:697 msgid "Your message" msgstr "Votre message" -#: agenda_culturel/models.py:471 agenda_culturel/views.py:343 +#: agenda_culturel/models.py:701 agenda_culturel/views.py:366 msgid "Closed" msgstr "Fermé" -#: agenda_culturel/models.py:471 +#: agenda_culturel/models.py:701 msgid "this message has been processed and no longer needs to be handled" msgstr "Ce message a été traité et ne nécessite plus d'être pris en charge" -#: agenda_culturel/models.py:472 +#: agenda_culturel/models.py:702 msgid "Comments" msgstr "Commentaires" -#: agenda_culturel/models.py:472 +#: agenda_culturel/models.py:702 msgid "Comments on the message from the moderation team" msgstr "Commentaires sur ce message par l'équipe de modération" -#: agenda_culturel/models.py:481 -msgid "Running" -msgstr "" +#: agenda_culturel/models.py:711 +msgid "ical" +msgstr "ical" -#: agenda_culturel/models.py:482 -msgid "Canceled" -msgstr "Annulé" +#: agenda_culturel/models.py:714 +msgid "simple" +msgstr "simple" -#: agenda_culturel/models.py:483 -msgid "Success" -msgstr "Succès" +#: agenda_culturel/models.py:715 +msgid "Headless Chromium" +msgstr "chromium sans interface" -#: agenda_culturel/models.py:484 -msgid "Failed" -msgstr "Erreur" +#: agenda_culturel/models.py:719 +msgid "daily" +msgstr "chaque jour" -#: agenda_culturel/models.py:489 +#: agenda_culturel/models.py:720 +msgid "weekly" +msgstr "chaque semaine" + +#: agenda_culturel/models.py:722 +msgid "Processor" +msgstr "Processeur" + +#: agenda_culturel/models.py:723 +msgid "Downloader" +msgstr "Téléchargeur" + +#: agenda_culturel/models.py:725 +msgid "Import recurrence" +msgstr "Récurrence d'import" + +#: agenda_culturel/models.py:728 msgid "Source" msgstr "Source" -#: agenda_culturel/models.py:489 +#: agenda_culturel/models.py:728 msgid "URL of the source document" msgstr "URL du document source" -#: agenda_culturel/models.py:490 +#: agenda_culturel/models.py:729 msgid "Browsable url" msgstr "URL navigable" -#: agenda_culturel/models.py:490 +#: agenda_culturel/models.py:729 msgid "URL of the corresponding document that will be shown to visitors." msgstr "URL correspondant au document et qui sera montrée aux visiteurs" -#: agenda_culturel/models.py:494 +#: agenda_culturel/models.py:731 +msgid "Address for each imported event" +msgstr "Adresse de chaque événement importé" + +#: agenda_culturel/models.py:733 +msgid "Category of each imported event" +msgstr "Catégorie de chaque événement importé" + +#: agenda_culturel/models.py:734 +msgid "Tags for each imported event" +msgstr "Étiquettes de chaque événement importé" + +#: agenda_culturel/models.py:734 +msgid "A list of tags that describe each imported event." +msgstr "Une liste d'étiquettes décrivant chaque événement importé" + +#: agenda_culturel/models.py:746 +msgid "Running" +msgstr "En cours" + +#: agenda_culturel/models.py:747 +msgid "Canceled" +msgstr "Annulé" + +#: agenda_culturel/models.py:748 +msgid "Success" +msgstr "Succès" + +#: agenda_culturel/models.py:749 +msgid "Failed" +msgstr "Erreur" + +#: agenda_culturel/models.py:754 +msgid "Recurrent import" +msgstr "Import récurrent" + +#: agenda_culturel/models.py:754 +msgid "Reference to the recurrent import processing" +msgstr "Référence du processus d'import récurrent" + +#: agenda_culturel/models.py:758 msgid "Error message" msgstr "Votre message" -#: agenda_culturel/models.py:496 +#: agenda_culturel/models.py:760 msgid "Number of collected events" msgstr "Nombre d'événements collectés" -#: agenda_culturel/models.py:497 +#: agenda_culturel/models.py:761 msgid "Number of imported events" msgstr "Nombre d'événements importés" -#: agenda_culturel/models.py:498 +#: agenda_culturel/models.py:762 msgid "Number of updated events" msgstr "Nombre d'événements mis à jour" -#: agenda_culturel/models.py:499 +#: agenda_culturel/models.py:763 msgid "Number of removed events" msgstr "Nombre d'événements supprimés" -#: agenda_culturel/settings/base.py:134 +#: agenda_culturel/settings/base.py:135 msgid "English" msgstr "anglais" -#: agenda_culturel/settings/base.py:135 +#: agenda_culturel/settings/base.py:136 msgid "French" msgstr "français" -#: agenda_culturel/views.py:190 +#: agenda_culturel/views.py:203 msgid "The static content has been successfully updated." msgstr "Le contenu statique a été modifié avec succès." -#: agenda_culturel/views.py:196 +#: agenda_culturel/views.py:209 msgid "The event has been successfully modified." msgstr "L'événement a été modifié avec succès." -#: agenda_culturel/views.py:207 +#: agenda_culturel/views.py:220 msgid "The event has been successfully deleted." msgstr "L'événement a été supprimé avec succès" -#: agenda_culturel/views.py:224 +#: agenda_culturel/views.py:247 msgid "The status has been successfully modified." msgstr "Le status a été modifié avec succès." -#: agenda_culturel/views.py:246 +#: agenda_culturel/views.py:269 msgid "The event is saved." msgstr "L'événement est enregistré." -#: agenda_culturel/views.py:249 +#: agenda_culturel/views.py:272 msgid "" "The event has been submitted and will be published as soon as it has been " "validated by the moderation team." @@ -358,7 +413,7 @@ msgstr "" "L'événement a été soumis et sera publié dès qu'il aura été validé par " "l'équipe de modération." -#: agenda_culturel/views.py:279 +#: agenda_culturel/views.py:302 msgid "" "The event has been successfully extracted, and you can now submit it after " "modifying it if necessary." @@ -366,7 +421,7 @@ msgstr "" "L'événement a été extrait avec succès, vous pouvez maintenant le soumettre " "après l'avoir modifié au besoin." -#: agenda_culturel/views.py:283 +#: agenda_culturel/views.py:306 msgid "" "Unable to extract an event from the proposed URL. Please use the form below " "to submit the event." @@ -374,12 +429,16 @@ msgstr "" "Impossible d'extraire un événement depuis l'URL proposée. Veuillez utiliser " "le formulaire ci-dessous pour soumettre l'événement." -#: agenda_culturel/views.py:292 +#: agenda_culturel/views.py:315 msgid "This URL has already been submitted, and you can find the event below." msgstr "" "Cette URL a déjà été soumise, et vous trouverez l'événement ci-dessous." -#: agenda_culturel/views.py:296 +#: agenda_culturel/views.py:319 +msgid "This URL has already been submitted and is awaiting moderation." +msgstr "Cette URL a déjà été soumise, et est en attente de modération" + +#: agenda_culturel/views.py:321 msgid "" "This URL has already been submitted, but has not been selected for " "publication by the moderation team." @@ -387,34 +446,87 @@ msgstr "" "Cette URL a déjà été soumise, mais n'a pas été retenue par l'équipe de " "modération pour la publication." -#: agenda_culturel/views.py:298 -msgid "This URL has already been submitted and is awaiting moderation." -msgstr "Cette URL a déjà été soumise, et est en attente de modération" - -#: agenda_culturel/views.py:320 +#: agenda_culturel/views.py:343 msgid "Your message has been sent successfully." msgstr "L'événement a été supprimé avec succès" -#: agenda_culturel/views.py:328 +#: agenda_culturel/views.py:351 msgid "The contact message properties has been successfully modified." msgstr "Les propriétés du message de contact ont été modifié avec succès." -#: agenda_culturel/views.py:343 +#: agenda_culturel/views.py:366 msgid "Open" msgstr "Ouvert" -#: agenda_culturel/views.py:383 +#: agenda_culturel/views.py:406 msgid "Search" msgstr "Rechercher" -#: agenda_culturel/views.py:491 +#: agenda_culturel/views.py:521 msgid "The import has been run successfully." msgstr "L'import a été lancé avec succès" -#: agenda_culturel/views.py:521 +#: agenda_culturel/views.py:537 msgid "The import has been canceled." msgstr "L'import a été annulé" +#: agenda_culturel/views.py:571 +msgid "The recurrent import has been successfully modified." +msgstr "L'import récurrent a été modifié avec succès." + +#: agenda_culturel/views.py:577 +#, fuzzy +#| msgid "The event has been successfully deleted." +msgid "The recurrent import has been successfully deleted." +msgstr "L'import récurrent a été supprimé avec succès" + +#: agenda_culturel/views.py:606 +#, fuzzy +#| msgid "The import has been canceled." +msgid "The import has been launched." +msgstr "L'import a été annulé" + +#: agenda_culturel/views.py:665 +msgid "The merge has been successfully completed." +msgstr "La fusion a été réalisée avec succès." + +#: agenda_culturel/views.py:694 +msgid "Events have been marked as unduplicated." +msgstr "Les événements ont été marqués comme non dupliqués." + +#: agenda_culturel/views.py:711 +msgid "" +"The selected event has been retained, while the other has been moved " +"to the recycle bin." +msgstr "" +"L'événement sélectionné a été conservé, l'autre a été déplacé dans la " +"corbeille." + +#: agenda_culturel/views.py:713 +msgid "" +"The selected event has been retained, while the others have been moved " +"to the recycle bin." +msgstr "" +"L'événement sélectionné a été conservé, les autres ont été déplacés dans la " +"corbeille." + +#: agenda_culturel/views.py:719 +msgid "The event has been withdrawn from the group and made independent." +msgstr "L'événement a été retiré du groupe et rendu indépendant." + +#: agenda_culturel/views.py:765 +msgid "The event was successfully duplicated." +msgstr "L'événement a été marqué dupliqué avec succès." + +#: agenda_culturel/views.py:768 +msgid "" +"The event has been successfully flagged as a duplicate. The moderation team " +"will deal with your suggestion shortly." +msgstr "" +"L'événement a été signalé comme dupliqué avec succès. Votre suggestion sera " +"prochainement prise en charge par l'équipe de modération." + + msgid "Add another" msgstr "Ajouter un autre" diff --git a/src/agenda_culturel/migrations/0028_recurrentimport_batchimportation_recurrentimport.py b/src/agenda_culturel/migrations/0028_recurrentimport_batchimportation_recurrentimport.py new file mode 100644 index 0000000..85b07a4 --- /dev/null +++ b/src/agenda_culturel/migrations/0028_recurrentimport_batchimportation_recurrentimport.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.7 on 2024-01-20 15:59 + +from django.db import migrations, models +import django.db.models.deletion +import django_better_admin_arrayfield.models.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('agenda_culturel', '0027_set_dtstart_dtend'), + ] + + operations = [ + migrations.CreateModel( + name='RecurrentImport', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('processor', models.CharField(choices=[('ical', 'ical')], default='ical', max_length=20, verbose_name='Processor')), + ('recurrence', models.CharField(choices=[('daily', 'daily'), ('weekly', 'weekly')], default='daily', max_length=10, verbose_name='Import recurrence')), + ('source', models.URLField(help_text='URL of the source document', max_length=1024, verbose_name='Source')), + ('browsable_url', models.URLField(blank=True, help_text='URL of the corresponding document that will be shown to visitors.', max_length=1024, null=True, verbose_name='Browsable url')), + ('defaultLocation', models.CharField(blank=True, help_text='Address for each imported event', max_length=512, null=True, verbose_name='Location')), + ('defaultTags', django_better_admin_arrayfield.models.fields.ArrayField(base_field=models.CharField(max_length=64), blank=True, help_text='A list of tags that describe each imported event.', null=True, size=None, verbose_name='Tags for each imported event')), + ('defaultCategory', models.ForeignKey(blank=True, help_text='Category of each imported event', null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.category', verbose_name='Category')), + ], + ), + migrations.AddField( + model_name='batchimportation', + name='recurrentImport', + field=models.ForeignKey(blank=True, editable=False, help_text='Reference to the recurrent import processing', null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.recurrentimport', verbose_name='Recurrent import'), + ), + ] diff --git a/src/agenda_culturel/migrations/0029_remove_batchimportation_browsable_url_and_more.py b/src/agenda_culturel/migrations/0029_remove_batchimportation_browsable_url_and_more.py new file mode 100644 index 0000000..7639342 --- /dev/null +++ b/src/agenda_culturel/migrations/0029_remove_batchimportation_browsable_url_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.7 on 2024-01-25 21:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('agenda_culturel', '0028_recurrentimport_batchimportation_recurrentimport'), + ] + + operations = [ + migrations.RemoveField( + model_name='batchimportation', + name='browsable_url', + ), + migrations.RemoveField( + model_name='batchimportation', + name='source', + ), + ] diff --git a/src/agenda_culturel/migrations/0030_recurrentimport_downloader.py b/src/agenda_culturel/migrations/0030_recurrentimport_downloader.py new file mode 100644 index 0000000..154769e --- /dev/null +++ b/src/agenda_culturel/migrations/0030_recurrentimport_downloader.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2024-01-26 10:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('agenda_culturel', '0029_remove_batchimportation_browsable_url_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='recurrentimport', + name='downloader', + field=models.CharField(choices=[('simple', 'simple'), ('chromium headless', 'Headless Chromium')], default='simple', max_length=20, verbose_name='Downloader'), + ), + ] diff --git a/src/agenda_culturel/models.py b/src/agenda_culturel/models.py index 4019e37..46c57c4 100644 --- a/src/agenda_culturel/models.py +++ b/src/agenda_culturel/models.py @@ -503,8 +503,9 @@ class Event(models.Model): for e in events: e.possibly_duplicated = group - # finally update all events (including current) - Event.objects.bulk_update(events + [self], fields=["possibly_duplicated"]) + # finally update all events (including current if already created) + elist = list(events) + ([self] if self.pk is not None else []) + Event.objects.bulk_update(elist, fields=["possibly_duplicated"]) def data_fields(all=False): @@ -631,7 +632,7 @@ class Event(models.Model): nb_updated = Event.objects.bulk_update(to_update, fields = Event.data_fields() + ["imported_date", "modified_date", "uuids", "status"]) nb_draft = 0 - if remove_missing_from_source is not None: + if remove_missing_from_source is not None and max_date is not None: # events that are missing from the import but in database are turned into drafts # only if they are in the future @@ -704,6 +705,41 @@ class ContactMessage(models.Model): return ContactMessage.objects.filter(closed=False).count() +class RecurrentImport(models.Model): + + class PROCESSOR(models.TextChoices): + ICAL = "ical", _("ical") + + class DOWNLOADER(models.TextChoices): + SIMPLE = "simple", _("simple") + CHROMIUMHEADLESS = "chromium headless", _("Headless Chromium") + + + class RECURRENCE(models.TextChoices): + DAILY = "daily", _("daily"), + WEEKLY = "weekly", _("weekly") + + processor = models.CharField(_("Processor"), max_length=20, choices=PROCESSOR.choices, default=PROCESSOR.ICAL) + downloader = models.CharField(_("Downloader"), max_length=20, choices=DOWNLOADER.choices, default=DOWNLOADER.SIMPLE) + + recurrence = models.CharField(_("Import recurrence"), max_length=10, choices=RECURRENCE.choices, default=RECURRENCE.DAILY) + + + source = models.URLField(verbose_name=_('Source'), help_text=_("URL of the source document"), max_length=1024) + browsable_url = models.URLField(verbose_name=_('Browsable url'), help_text=_("URL of the corresponding document that will be shown to visitors."), max_length=1024, blank=True, null=True) + + defaultLocation = models.CharField(verbose_name=_('Location'), help_text=_('Address for each imported event'), max_length=512, null=True, blank=True) + + defaultCategory = models.ForeignKey(Category, verbose_name=_('Category'), help_text=_('Category of each imported event'), blank=True, null=True, on_delete=models.SET_NULL) + defaultTags = ArrayField(models.CharField(max_length=64), verbose_name=_('Tags for each imported event'), help_text=_("A list of tags that describe each imported event."), blank=True, null=True) + + def nb_imports(self): + return BatchImportation.objects.filter(recurrentImport=self).count() + + def get_absolute_url(self): + return reverse("view_rimport", kwargs={"pk": self.pk}) + + class BatchImportation(models.Model): class STATUS(models.TextChoices): @@ -715,8 +751,7 @@ class BatchImportation(models.Model): created_date = models.DateTimeField(auto_now_add=True) - source = models.URLField(verbose_name=_('Source'), help_text=_("URL of the source document"), max_length=1024, blank=True, null=True) - browsable_url = models.URLField(verbose_name=_('Browsable url'), help_text=_("URL of the corresponding document that will be shown to visitors."), max_length=1024, blank=True, null=True) + recurrentImport = models.ForeignKey(RecurrentImport, verbose_name=_('Recurrent import'), help_text=_('Reference to the recurrent import processing'), blank=True, null=True, on_delete=models.SET_NULL, editable=False) status = models.CharField(_("Status"), max_length=20, choices=STATUS.choices, default=STATUS.RUNNING) diff --git a/src/agenda_culturel/static/style.scss b/src/agenda_culturel/static/style.scss index 8bac42e..cc97c3d 100644 --- a/src/agenda_culturel/static/style.scss +++ b/src/agenda_culturel/static/style.scss @@ -774,4 +774,10 @@ article>article { .strike { text-decoration: line-through; +} + +.buttons { + display: flex; + flex: auto; + gap: 0.4em; } \ No newline at end of file diff --git a/src/agenda_culturel/templates/agenda_culturel/batch-imports-inc.html b/src/agenda_culturel/templates/agenda_culturel/batch-imports-inc.html new file mode 100644 index 0000000..def1711 --- /dev/null +++ b/src/agenda_culturel/templates/agenda_culturel/batch-imports-inc.html @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + {% for obj in paginator_filter %} + + + + + + + + + + + {% endfor %} + +
IdentifiantDateStatusActionévénements
initialimportésmis à jourdépubliés
{{ obj.id }}{{ obj.created_date }}{{ obj.status }}{% if obj.status == "running" %}Annuler{% endif %}{% if obj.status == "success" %}{{ obj.nb_initial }}{% endif %}{% if obj.status == "success" %}{{ obj.nb_imported }}{% endif %}{% if obj.status == "success" %}{{ obj.nb_updated }}{% endif %}{% if obj.status == "success" %}{{ obj.nb_removed }}{% endif %}
+
+ + {% if paginator_filter.has_previous %} + « premier + précédent + {% endif %} + + + Page {{ paginator_filter.number }} sur {{ paginator_filter.paginator.num_pages }} + + + {% if paginator_filter.has_next %} + suivant + dernier » + {% endif %} + +
\ No newline at end of file diff --git a/src/agenda_culturel/templates/agenda_culturel/batchimportation_form.html b/src/agenda_culturel/templates/agenda_culturel/batchimportation_form.html index 903a165..d243ab1 100644 --- a/src/agenda_culturel/templates/agenda_culturel/batchimportation_form.html +++ b/src/agenda_culturel/templates/agenda_culturel/batchimportation_form.html @@ -1,12 +1,12 @@ {% extends "agenda_culturel/page.html" %} {% load static %} -{% block title %}Importation par lot{% endblock %} +{% block title %}Importation manuelle{% endblock %} {% block content %} -

Importation par lot

+

Importation manuelle

{% csrf_token %} diff --git a/src/agenda_culturel/templates/agenda_culturel/imports.html b/src/agenda_culturel/templates/agenda_culturel/imports.html index eadb3cb..24d3ef0 100644 --- a/src/agenda_culturel/templates/agenda_culturel/imports.html +++ b/src/agenda_culturel/templates/agenda_culturel/imports.html @@ -12,58 +12,11 @@
- Nouvel import + Import manuel

Importations par lot

- - - - - - - - - - - - - - - - - - {% for obj in paginator_filter %} - - - - - - - - - - - {% endfor %} - -
IdentifiantDateStatusActionévénements
initialimportésmis à jourdépubliés
{{ obj.id }}{{ obj.created_date }}{{ obj.status }}{% if obj.status == "running" %}Annuler{% endif %}{% if obj.status == "success" %}{{ obj.nb_initial }}{% endif %}{% if obj.status == "success" %}{{ obj.nb_imported }}{% endif %}{% if obj.status == "success" %}{{ obj.nb_updated }}{% endif %}{% if obj.status == "success" %}{{ obj.nb_removed }}{% endif %}
-
- - {% if paginator_filter.has_previous %} - « premier - précédent - {% endif %} - - - Page {{ paginator_filter.number }} sur {{ paginator_filter.paginator.num_pages }} - - - {% if paginator_filter.has_next %} - suivant - dernier » - {% endif %} - -
+ {% include "agenda_culturel/batch-imports-inc.html" with paginator_filter=paginator_filter %}
{% include "agenda_culturel/side-nav.html" with current="imports" %} diff --git a/src/agenda_culturel/templates/agenda_culturel/page-rimport.html b/src/agenda_culturel/templates/agenda_culturel/page-rimport.html new file mode 100644 index 0000000..2976233 --- /dev/null +++ b/src/agenda_culturel/templates/agenda_culturel/page-rimport.html @@ -0,0 +1,53 @@ +{% extends "agenda_culturel/page.html" %} + +{% block title %}Import récurrent #{{ object.pk }}{% endblock %} + +{% load cat_extra %} +{% load utils_extra %} + +{% block entete_header %} + {% css_categories %} +{% endblock %} + + +{% block content %} +
+
+
+ < Retour +
+ Exécuter {% picto_from_name "download-cloud" %} + Éditer {% picto_from_name "edit" %} + Supprimer {% picto_from_name "trash" %} +
+ +

Import récurrent #{{ object.pk }}

+
    +
  • Processeur : {{ object.processor }}
    • +
  • Téléchargeur : {{ object.downloader }}
    • +
  • Recurrence : {{ object.recurrence }}
    • +
  • Source : {{ object.source }}
    • +
  • Adresse naviguable : {{ object.browsable_url }}
    • +
  • Valeurs par défaut : +
      +
    • Localisation : {{ object.defaultLocation }}
      • +
    • Catégorie : {{ object.defaultCategory }}
      • +
    • Étiquettes : + {% for tag in object.defaultTags %} + {{ tag }}{% if not forloop.last %}, {% endif %} + {% endfor %} +
      • +
    +
    • +
+ +
+ + {% include "agenda_culturel/batch-imports-inc.html" with paginator_filter=paginator_filter %} + +
+ + + {% include "agenda_culturel/side-nav.html" with current="rimports" %} +
+{% endblock %} \ No newline at end of file diff --git a/src/agenda_culturel/templates/agenda_culturel/recurrentimport_confirm_delete.html b/src/agenda_culturel/templates/agenda_culturel/recurrentimport_confirm_delete.html new file mode 100644 index 0000000..6850e3f --- /dev/null +++ b/src/agenda_culturel/templates/agenda_culturel/recurrentimport_confirm_delete.html @@ -0,0 +1,18 @@ +{% extends "agenda_culturel/page.html" %} + +{% block title %}Supprimer {{ object.title }}{% endblock %} + + +{% block content %} + +

Suppression de l'import récurrent {{ object.pk }}

+{% csrf_token %} +

Êtes-vous sûr·e de vouloir supprimer l'import récurrent « {{ object.pk }} »  correspondant à la source {{ object.source }} ?

+ {{ form }} +
+ Annuler + +
+ + +{% endblock %} \ No newline at end of file diff --git a/src/agenda_culturel/templates/agenda_culturel/recurrentimport_form.html b/src/agenda_culturel/templates/agenda_culturel/recurrentimport_form.html new file mode 100644 index 0000000..e195de0 --- /dev/null +++ b/src/agenda_culturel/templates/agenda_culturel/recurrentimport_form.html @@ -0,0 +1,27 @@ +{% extends "agenda_culturel/page.html" %} +{% load static %} + +{% block title %}Importation récurrente{% endblock %} + +{% block entete_header %} + + + + +{% endblock %} + +{% block content %} + +

Importation récurrente

+ +
+
{% csrf_token %} + {{ form.as_p }} +
+ Annuler + +
+
+
+ +{% endblock %} \ No newline at end of file diff --git a/src/agenda_culturel/templates/agenda_culturel/rimports.html b/src/agenda_culturel/templates/agenda_culturel/rimports.html new file mode 100644 index 0000000..a9515c6 --- /dev/null +++ b/src/agenda_culturel/templates/agenda_culturel/rimports.html @@ -0,0 +1,75 @@ +{% extends "agenda_culturel/page.html" %} + +{% block title %}Importations récurrentes{% endblock %} + +{% load utils_extra %} +{% load cat_extra %} +{% block entete_header %} + {% css_categories %} +{% endblock %} + +{% block content %} +
+
+
+ Ajouter {% picto_from_name "plus-circle" %} +

Importations récurrentes

+
+ + + + + + + + + + + + + + + + {% for obj in paginator_filter %} + + + + + + + + + + + {% endfor %} + +
IdentifiantProcesseurTéléchargeurRécurrenceSourceAdresse naviguableNombre d'importsActions
{{ obj.pk }}{{ obj.processor }}{{ obj.downloader }}{{ obj.recurrence }}{{ obj.source|truncatechars_middle:32 }}{{ obj.browsable_url|truncatechars_middle:32 }}{{ obj.nb_imports }} +
+ Exécuter + Consulter +
+
+ +
+ + {% if paginator_filter.has_previous %} + « premier + précédent + {% endif %} + + + Page {{ paginator_filter.number }} sur {{ paginator_filter.paginator.num_pages }} + + + {% if paginator_filter.has_next %} + suivant + dernier » + {% endif %} + +
+
+ +{% include "agenda_culturel/side-nav.html" with current="rimports" %} +
+ +{% endblock %} \ No newline at end of file diff --git a/src/agenda_culturel/templates/agenda_culturel/run_rimport_confirm.html b/src/agenda_culturel/templates/agenda_culturel/run_rimport_confirm.html new file mode 100644 index 0000000..686982e --- /dev/null +++ b/src/agenda_culturel/templates/agenda_culturel/run_rimport_confirm.html @@ -0,0 +1,25 @@ +{% extends "agenda_culturel/page.html" %} + +{% block title %}Lancer l'import #{{ object.pk }}{% endblock %} + + +{% block content %} + +
+
+

Lancement de l'import #{{ object.pk }}

+
+
{% csrf_token %} +

Êtes-vous sûr·e de vouloir lancer l'import récurrent « {{ object.pk }} »  correspondant à la source {{ object.source }} ?

+ {{ form }} + + +
+
+ +{% endblock %} \ No newline at end of file diff --git a/src/agenda_culturel/templates/agenda_culturel/side-nav.html b/src/agenda_culturel/templates/agenda_culturel/side-nav.html index dc1dba9..22bc65c 100644 --- a/src/agenda_culturel/templates/agenda_culturel/side-nav.html +++ b/src/agenda_culturel/templates/agenda_culturel/side-nav.html @@ -21,7 +21,17 @@
  • Derniers événements soumis{% show_badges_events "left" %}
  • Messages de contact{% show_badge_contactmessages "left" %}
  • Gestion des doublons{% show_badge_duplicated "left" %}
    • -
  • Importations par lot
    • + + +
    + diff --git a/src/agenda_culturel/templatetags/utils_extra.py b/src/agenda_culturel/templatetags/utils_extra.py index 1e79e86..51e53ef 100644 --- a/src/agenda_culturel/templatetags/utils_extra.py +++ b/src/agenda_culturel/templatetags/utils_extra.py @@ -1,5 +1,6 @@ from django import template from django.utils.safestring import mark_safe +from django.template.defaultfilters import stringfilter from urllib.parse import urlparse from datetime import timedelta, date @@ -71,3 +72,15 @@ def picto_from_name(name, datatooltip=""): @register.filter def int_to_abc(d): return auc[int(d)] + +@register.filter(is_safe=True) +@stringfilter +def truncatechars_middle(value, arg): + try: + ln = int(arg) + except ValueError: + return value + if len(value) <= ln: + return value + else: + return '{}...{}'.format(value[:ln//2], value[-((ln+1)//2):]) \ No newline at end of file diff --git a/src/agenda_culturel/urls.py b/src/agenda_culturel/urls.py index 77f6490..d5338e7 100644 --- a/src/agenda_culturel/urls.py +++ b/src/agenda_culturel/urls.py @@ -38,8 +38,14 @@ urlpatterns = [ path('contactmessages', contactmessages, name='contactmessages'), path('contactmessage/', ContactMessageUpdateView.as_view(), name='contactmessage'), path("imports/", imports, name="imports"), - path("imports/add", BatchImportationCreateView.as_view(), name="add_import"), + path("imports/add", add_import, name="add_import"), path("imports//cancel", cancel_import, name="cancel_import"), + path("rimports/", recurrent_imports, name="recurrent_imports"), + path("rimports/add", RecurrentImportCreateView.as_view(), name="add_rimport"), + path("rimports//view", view_rimport, name="view_rimport"), + path("rimports//edit", RecurrentImportUpdateView.as_view(), name="edit_rimport"), + path("rimports//delete", RecurrentImportDeleteView.as_view(), name="delete_rimport"), + path("rimports//run", run_rimport, name="run_rimport"), path("duplicates/", duplicates, name="duplicates"), path("duplicates/", DuplicatedEventsDetailView.as_view(), name="view_duplicate"), path("duplicates//fix", fix_duplicate, name="fix_duplicate"), diff --git a/src/agenda_culturel/views.py b/src/agenda_culturel/views.py index 2813b32..79ca65b 100644 --- a/src/agenda_culturel/views.py +++ b/src/agenda_culturel/views.py @@ -11,9 +11,9 @@ from django.http import HttpResponseRedirect from django.urls import reverse import urllib -from .forms import EventSubmissionForm, EventForm, BatchImportationForm, FixDuplicates, SelectEventInList, MergeDuplicates +from .forms import EventSubmissionForm, EventForm, BatchImportationForm, FixDuplicates, SelectEventInList, MergeDuplicates, RecurrentImportForm -from .models import Event, Category, StaticContent, ContactMessage, BatchImportation, DuplicatedEvents +from .models import Event, Category, StaticContent, ContactMessage, BatchImportation, DuplicatedEvents, RecurrentImport from django.utils import timezone from enum import StrEnum from datetime import date, timedelta @@ -32,7 +32,7 @@ from django.contrib.messages.views import SuccessMessageMixin from .calendar import CalendarMonth, CalendarWeek, CalendarDay from .extractors import ExtractorAllURLs -from .celery import app as celery_app, import_events_from_json, import_events_from_url +from .celery import app as celery_app, import_events_from_json, run_recurrent_import import unicodedata import logging @@ -507,28 +507,21 @@ def imports(request): return render(request, 'agenda_culturel/imports.html', {'paginator_filter': response} ) -class BatchImportationCreateView(SuccessMessageMixin, LoginRequiredMixin, CreateView): - model = BatchImportation +@login_required(login_url="/accounts/login/") +def add_import(request): + form = BatchImportationForm() - success_url = reverse_lazy('imports') - success_message = _('The import has been run successfully.') + if request.method == 'POST': + form = BatchImportationForm(request.POST) - form_class = BatchImportationForm + if form.is_valid(): - - def form_valid(self, form): - - # run import - if "json" in form.data and form.data["json"] is not None and form.data["json"].strip() != "": result = import_events_from_json.delay(form.data["json"]) - else: - result = import_events_from_url.delay(form.data["source"], form.data["browsable_url"]) - # update the object with celery_id - form.instance.celery_id = result.id - - return super().form_valid(form) + messages.success(request, _("The import has been run successfully.")) + return HttpResponseRedirect(reverse_lazy("imports")) + return render(request, 'agenda_culturel/batchimportation_form.html', {"form": form}) @login_required(login_url="/accounts/login/") @@ -547,6 +540,78 @@ def cancel_import(request, pk): cancel_url = reverse_lazy("imports") return render(request, 'agenda_culturel/cancel_import_confirm.html', {"object": import_process, "cancel_url": cancel_url}) +######################### +## recurrent importations +######################### + +@login_required(login_url="/accounts/login/") +def recurrent_imports(request): + paginator = Paginator(RecurrentImport.objects.all().order_by("-pk"), 10) + page = request.GET.get('page') + + try: + response = paginator.page(page) + except PageNotAnInteger: + response = paginator.page(1) + except EmptyPage: + response = paginator.page(paginator.num_pages) + + return render(request, 'agenda_culturel/rimports.html', {'paginator_filter': response} ) + + +class RecurrentImportCreateView(LoginRequiredMixin, CreateView): + model = RecurrentImport + success_url = reverse_lazy('recurrent_imports') + form_class = RecurrentImportForm + + +class RecurrentImportUpdateView(SuccessMessageMixin, LoginRequiredMixin, UpdateView): + model = RecurrentImport + form_class = RecurrentImportForm + success_message = _('The recurrent import has been successfully modified.') + + +class RecurrentImportDeleteView(SuccessMessageMixin, LoginRequiredMixin, DeleteView): + model = RecurrentImport + success_url = reverse_lazy('recurrent_imports') + success_message = _('The recurrent import has been successfully deleted.') + + +@login_required(login_url="/accounts/login/") +def view_rimport(request, pk): + obj = get_object_or_404(RecurrentImport, pk=pk) + paginator = Paginator(BatchImportation.objects.filter(recurrentImport=pk).order_by("-created_date"), 10) + page = request.GET.get('page') + + try: + response = paginator.page(page) + except PageNotAnInteger: + response = paginator.page(1) + except EmptyPage: + response = paginator.page(paginator.num_pages) + + return render(request, 'agenda_culturel/page-rimport.html', {'paginator_filter': response, 'object': obj} ) + + + +@login_required(login_url="/accounts/login/") +def run_rimport(request, pk): + rimport = get_object_or_404(RecurrentImport, pk=pk) + + if request.method == 'POST': + + # run recurrent import + result = run_recurrent_import.delay(pk) + + messages.success(request, _("The import has been launched.")) + return HttpResponseRedirect(reverse_lazy("view_rimport", args=[pk])) + else: + return render(request, 'agenda_culturel/run_rimport_confirm.html', {"object": rimport }) + +######################### +## duplicated events +######################### + class DuplicatedEventsDetailView(LoginRequiredMixin, DetailView): model = DuplicatedEvents @@ -597,7 +662,7 @@ def merge_duplicate(request, pk): e.status = Event.STATUS.TRASH Event.objects.bulk_update(events, fields=["status"]) - messages.info(request, _("La fusion a été réalisée avec succès.")) + messages.info(request, _("The merge has been successfully completed.")) return HttpResponseRedirect(new_event.get_absolute_url()) return render(request, 'agenda_culturel/merge_duplicate.html', context={'form': form, 'object': edup}) @@ -626,7 +691,7 @@ def fix_duplicate(request, pk): s_event = events[0] date = s_event.start_day - messages.success(request, _("Les événements ont été marqués comme non dupliqués.")) + messages.success(request, _("Events have been marked as unduplicated.")) edup.delete() if date is None: return HttpResponseRedirect(reverse_lazy("home")) @@ -643,15 +708,15 @@ def fix_duplicate(request, pk): url = selected.get_absolute_url() edup.delete() if nb == 1: - messages.success(request, _("L'événement sélectionné a été conservé, l'autre a été déplacé dans la corbeille.")) + messages.success(request, _("The selected event has been retained, while the other has been moved to the recycle bin.")) else: - messages.success(request, _("L'événement sélectionné a été conservé, les autres ont été déplacés dans la corbeille.")) + messages.success(request, _("The selected event has been retained, while the others have been moved to the recycle bin.")) return HttpResponseRedirect(url) elif form.is_action_remove(): event = form.get_selected_event(edup) event.possibly_duplicated = None event.save() - messages.success(request, _("L'événement a été retiré du groupe et rendu indépendant.")) + messages.success(request, _("The event has been withdrawn from the group and made independent.")) if edup.nb_duplicated() == 1: return HttpResponseRedirect(event.get_absolute_url()) else: @@ -697,10 +762,10 @@ def set_duplicate(request, year, month, day, pk): event.set_possibly_duplicated(selected) event.save() if request.user.is_authenticated: - messages.success(request, _("L'événement a été marqué dupliqué avec succès.")) + messages.success(request, _("The event was successfully duplicated.")) return HttpResponseRedirect(reverse_lazy("view_duplicate", args=[event.possibly_duplicated.pk])) else: - messages.info(request, _("L'événement a été signalé comme dupliqué avec succès. Votre suggestion sera prochainement prise en charge par l'équipe de modération.")) + messages.info(request, _("The event has been successfully flagged as a duplicate. The moderation team will deal with your suggestion shortly.")) return HttpResponseRedirect(event.get_absolute_url()) diff --git a/src/requirements.txt b/src/requirements.txt index e0f2161..62a4233 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -33,3 +33,6 @@ django-compressor==4.4 django-libsass==0.9 django-ckeditor==6.7.0 django-recurrence==1.11.1 +icalendar==5.0.11 +lxml==5.1.0 +