On ajoute une fonction de suppression de cache

Fix #136

Makefile

@@ -55,6 +55,10 @@ create-categories:
 	docker exec -it $(BACKEND_APP_NAME) $(SHELL) "-c" \
 	"python3 manage.py runscript create_categories"
 
+create-reference-locations:
+	docker exec -it $(BACKEND_APP_NAME) $(SHELL) "-c" \
+	"python3 manage.py runscript create_reference_locations"
+
 build-dev:
 	DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose -f docker-compose.yml up --build -d
 

README.md

@@ -15,6 +15,12 @@ On peut aussi peupler les catégories avec un choix de catégories élémentaire
 
 * ```make create-categories```
 
+On peut aussi peupler les positions de référence qui serviront aux recherches géographiques avec la commande, après avoir éventuellement modifié le fichier [communes.json](./src/scripts/communes.json) qui contient pour l'exemple toutes les communes récupérées depuis [public.opendatasoft.com](https://public.opendatasoft.com/explore/dataset/georef-france-commune/export/?flg=fr-fr&disjunctive.reg_name&disjunctive.dep_name&disjunctive.arrdep_name&disjunctive.ze2020_name&disjunctive.bv2022_name&disjunctive.epci_name&disjunctive.ept_name&disjunctive.com_name&disjunctive.ze2010_name&disjunctive.com_is_mountain_area&sort=year&refine.dep_name=Puy-de-D%C3%B4me&location=9,45.51597,3.05969&basemap=jawg.light):
+
+* ```make create-reference-locations```
+
+
+
 ## Notes aux développeurs
 
 ### Ajout d'une nouvelle source *custom*
@@ -25,4 +31,14 @@ Pour ajouter une nouvelle source custom:
 - quand l'import fonctionne de manière indépendante dans ces expérimentations, il est tant de l'ajouter au site internet:
     - ajouter à la classe ```RecurrentImport.PROCESSOR``` présente dans le fichier ```src/agenda_culturel/models.py``` une entrée correspondant à cette source pour qu'elle soit proposée aux utilisateurs
     - ajouter à la fonction ```run_recurrent_import``` présente dans le fichier ```src/agenda_culturel/celery.py``` le test correspondant à cet ajout, pour lancer le bon extracteur
-- se rendre sur le site, page administration, et ajouter un import récurrent correspondant à cette nouvelle source
+- se rendre sur le site, page administration, et ajouter un import récurrent correspondant à cette nouvelle source
+
+### Récupérer un dump du prod sur un serveur dev
+
+* sur le serveur de dev:
+  * ```docker exec -i agenda_culturel-backend python3 manage.py dumpdata --natural-foreign --natural-primary --format=json --exclude=admin.logentry --indent=2   > fixtures/postgres-backup-20241101.json``` (à noter qu'ici on oublie les comptes, qu'il faudra recréer)
+* sur le serveur de prod:
+  * On récupère le dump json ```scp $SERVEUR:$PATH/fixtures/postgres-backup-20241101.json src/fixtures/```
+  * ```scripts/reset-database.sh FIXTURE COMMIT``` où ```FIXTURE``` est le timestamp dans le nom de la fixture, et ```COMMIT``` est l'ID du commit git correspondant à celle en prod sur le serveur au moment de la création de la fixture
+
+À noter que les images ne sont pas récupérées.

deployment/Dockerfile

@@ -5,10 +5,11 @@ WORKDIR /usr/src/app
 
 RUN --mount=type=cache,target=/var/cache/apt \
 	apt-get update && \
-    apt-get install --no-install-recommends -y build-essential libpq-dev gettext chromium-driver gdal-bin \
+    apt-get install --no-install-recommends -y build-essential libpq-dev gettext chromium-driver gdal-bin fonts-symbola \
     && rm -rf /var/lib/apt/lists/*
 
 
+
 COPY src/requirements.txt ./requirements.txt
 
 RUN --mount=type=cache,target=/root/.cache/pip \

deployment/scripts/nginx/nginx.conf

@@ -32,7 +32,7 @@ http {
         error_page 502 /static/html/500.html;
         error_page 503 /static/html/500.html;
 
-	if ($http_user_agent ~* "(?:Amazonbot)") {
+	if ($http_user_agent ~* (Amazonbot|meta-externalagent|ClaudeBot)) {
 	    return 444;
 	}
 

experimentations/get_arachnee_events.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+# coding: utf-8
+
+import os
+import json
+import sys
+
+# getting the name of the directory
+# where the this file is present.
+current = os.path.dirname(os.path.realpath(__file__))
+ 
+# Getting the parent directory name
+# where the current directory is present.
+parent = os.path.dirname(current)
+ 
+# adding the parent directory to 
+# the sys.path.
+sys.path.append(parent)
+
+from src.agenda_culturel.import_tasks.downloader import *
+from src.agenda_culturel.import_tasks.extractor import *
+from src.agenda_culturel.import_tasks.importer import *
+from src.agenda_culturel.import_tasks.custom_extractors import *
+
+
+if __name__ == "__main__":
+
+    u2e = URL2Events(ChromiumHeadlessDownloader(), arachnee.CExtractor())
+    url = "https://www.arachnee-concerts.com/wp-admin/admin-ajax.php?action=movies-filter&per_page=9999&date=NaN.NaN.NaN&theatres=Clermont-Fd&cat=&sorting=&list_all_events=&current_page="
+    url_human = "https://www.arachnee-concerts.com/agenda-des-concerts/Clermont-Fd/"
+
+    try:
+        events = u2e.process(url, url_human, cache = "cache-arachnee.html", default_values = {}, published = True)
+
+        exportfile = "events-arachnee.json"
+        print("Saving events to file {}".format(exportfile))
+        with open(exportfile, "w") as f:
+            json.dump(events, f, indent=4, default=str)
+    except Exception as e:
+        print("Exception: " + str(e))

experimentations/get_lacoope_events.py

@@ -33,7 +33,7 @@ if __name__ == "__main__":
     url_human = "https://www.lacoope.org/concerts-calendrier/"
 
     try:
-        events = u2e.process(url, url_human, cache = "cache-lacoope.html", default_values = {"category": "Concert", "location": "La Coopérative"}, published = True)
+        events = u2e.process(url, url_human, cache = "cache-lacoope.html", default_values = {"category": "Fêtes & Concerts", "location": "La Coopérative"}, published = True)
 
         exportfile = "events-lacoope.json"
         print("Saving events to file {}".format(exportfile))

experimentations/get_le_rio.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+# coding: utf-8
+
+import os
+import json
+import sys
+
+# getting the name of the directory
+# where the this file is present.
+current = os.path.dirname(os.path.realpath(__file__))
+ 
+# Getting the parent directory name
+# where the current directory is present.
+parent = os.path.dirname(current)
+ 
+# adding the parent directory to 
+# the sys.path.
+sys.path.append(parent)
+
+from src.agenda_culturel.import_tasks.downloader import *
+from src.agenda_culturel.import_tasks.extractor import *
+from src.agenda_culturel.import_tasks.importer import *
+from src.agenda_culturel.import_tasks.custom_extractors import *
+
+
+
+
+
+if __name__ == "__main__":
+
+    u2e = URL2Events(SimpleDownloader(), lerio.CExtractor())
+    url = "https://www.cinemalerio.com/evenements/"
+    url_human = "https://www.cinemalerio.com/evenements/"
+
+    try:
+        events = u2e.process(url, url_human, cache = "cache-le-rio.html", default_values = {"location": "Cinéma le Rio", "category": "Cinéma"}, published = True)
+
+        exportfile = "events-le-roi.json"
+        print("Saving events to file {}".format(exportfile))
+        with open(exportfile, "w") as f:
+            json.dump(events, f, indent=4, default=str)
+    except Exception as e:
+        print("Exception: " + str(e))

scripts/reset-database.sh

@@ -0,0 +1,92 @@
+#!/bin/sh
+
+
+FIXTURE=$1
+COMMIT=$2
+FORCE=$3
+
+help() {
+    echo "USAGE: scripts/reset-database.sh [FIXTURE] [COMMIT]"
+    echo " "
+    echo "Parameters:"
+    echo "   FIXTURE      A timestamp used in fixture name"
+    echo "   COMMIT       A commit ID used by git checkout"
+    echo " "
+    echo "Example:"
+    echo " scripts/reset-database.sh 20241110 cb69ece6ca5ba04e94dcc2758f53869c70224592"
+}
+
+bold=$(tput bold)
+normal=$(tput sgr0)
+echobold() {
+    echo "${bold}$1${normal}"
+}
+
+if ! [ -n "$FORCE" ]; then
+    nginx=`docker ps|grep nginx`
+    if [ -n "$nginx" ]; then
+        echo "WARNING: this script is probably run on a production server. Use a third parameter if you really want to run it."
+        exit 3
+    fi
+fi
+
+if ! [ -n "$FIXTURE" ]; then
+    echo "No fixture defined. Abort."
+    help
+    exit 1
+fi
+
+if ! [ -n "$COMMIT" ]; then
+    echo "No commit version defined. Abort."
+    help
+    exit 1
+fi
+
+
+FFILE=fixtures/postgres-backup-$FIXTURE.json
+
+if ! [ -f "src/$FFILE" ]; then
+    echo "ERROR: missing fixture file ($FFILE)"
+    exit 2
+fi
+
+
+
+
+echo " "
+echobold "WARNING: use Ctrl+C to stop the reset process since a 'no' answer cannot be detected."
+echo " "
+
+# remove all elements in database
+echobold "Flush database"
+docker exec -i agenda_culturel-backend python3 manage.py flush
+
+# move back database structure to the original
+echobold "Setup database structure to zero"
+docker exec -i agenda_culturel-backend python3 manage.py migrate agenda_culturel zero
+
+# reset code depending on a specific commit
+echobold "Move back to the desired commit"
+git checkout $COMMIT
+
+# change database to reach this specific version
+echobold "Setup database stucture according to the selected commit"
+docker exec -i agenda_culturel-backend python3 manage.py migrate agenda_culturel
+
+# remove all elements in database
+echobold "Flush database"
+docker exec -i agenda_culturel-backend python3 manage.py flush --no-input
+
+# import data
+echobold "Import data"
+docker exec -i agenda_culturel-backend python3 manage.py loaddata --format=json $FFILE
+
+# reset code to uptodate version
+echobold "Move back to last commit"
+git checkout main
+
+# update database structure
+echobold "Update database"
+docker exec -i agenda_culturel-backend python3 manage.py migrate agenda_culturel
+
+

src/agenda_culturel/admin.py

@@ -3,12 +3,15 @@ from django import forms
 from .models import (
     Event,
     Category,
+    Tag,
     StaticContent,
     DuplicatedEvents,
     BatchImportation,
     RecurrentImport,
     Place,
-    ContactMessage
+    Message,
+    ReferenceLocation,
+    Organisation
 )
 from django_better_admin_arrayfield.admin.mixins import DynamicArrayMixin
 from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget
@@ -16,12 +19,15 @@ from django_better_admin_arrayfield.models.fields import DynamicArrayField
 
 
 admin.site.register(Category)
+admin.site.register(Tag)
 admin.site.register(StaticContent)
 admin.site.register(DuplicatedEvents)
 admin.site.register(BatchImportation)
 admin.site.register(RecurrentImport)
 admin.site.register(Place)
-admin.site.register(ContactMessage)
+admin.site.register(Message)
+admin.site.register(ReferenceLocation)
+admin.site.register(Organisation)
 
 
 class URLWidget(DynamicArrayWidget):

src/agenda_culturel/calendar.py

@@ -1,10 +1,14 @@
 from datetime import datetime, timedelta, date, time
 import calendar
-from django.db.models import Q
+from django.db.models import Q, F
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from django.template.defaultfilters import date as _date
 
+from django.db.models import CharField
+from django.db.models.functions import Lower
+
+CharField.register_lookup(Lower)
 
 import logging
 
@@ -113,6 +117,23 @@ class DayInCalendar:
             if e.start_time is None
             else e.start_time
         )
+        self.today_night = False
+        if self.is_today():
+            self.today_night = True
+            now = timezone.now()
+            nday = now.date()
+            ntime = now.time()
+            found = False
+            for idx,e in enumerate(self.events):
+                if (nday < e.start_day) or (nday == e.start_day and e.start_time and ntime <= e.start_time):
+                    self.events[idx].is_first_after_now = True
+                    found = True
+                    break
+            if found:
+                self.today_night = False
+
+    def is_today_after_events(self):
+        return self.is_today() and self.today_night
 
     def events_by_category_ordered(self):
         from .models import Category
@@ -171,11 +192,13 @@ class IntervalInDay(DayInCalendar):
         self.id = self.id + '-' + str(id)
 
 class CalendarList:
-    def __init__(self, firstdate, lastdate, filter=None, exact=False):
+    def __init__(self, firstdate, lastdate, filter=None, exact=False, ignore_dup=None, qs=None):
         self.firstdate = firstdate
         self.lastdate = lastdate
         self.now = date.today()
         self.filter = filter
+        self.ignore_dup = ignore_dup
+        self.qs = qs
 
         if exact:
             self.c_firstdate = self.firstdate
@@ -214,14 +237,20 @@ class CalendarList:
 
     def fill_calendar_days(self):
         if self.filter is None:
-            from .models import Event
+            if self.qs is None:
+                from .models import Event
 
-            qs = Event.objects.all()
+                qs = Event.objects.all()
+            else:
+                qs = self.qs
         else:
             qs = self.filter.qs
+
+        if self.ignore_dup:
+            qs = qs.exclude(other_versions=self.ignore_dup)
         startdatetime = timezone.make_aware(datetime.combine(self.c_firstdate, time.min), timezone.get_default_timezone())
         lastdatetime = timezone.make_aware(datetime.combine(self.c_lastdate, time.max), timezone.get_default_timezone())
-        self.events = qs.filter(
+        qs = qs.filter(
             (Q(recurrence_dtend__isnull=True) & Q(recurrence_dtstart__lte=lastdatetime))
             | (
                 Q(recurrence_dtend__isnull=False)
@@ -230,7 +259,15 @@ class CalendarList:
                     | Q(recurrence_dtend__lt=startdatetime)
                 )
             )
-        ).order_by("start_time").prefetch_related("exact_location").prefetch_related("category")
+            | (Q(start_day__lte=self.c_firstdate) & (Q(end_day__isnull=True) | Q(end_day__gte=self.c_firstdate)))
+        ).filter(
+            Q(other_versions__isnull=True) |
+            Q(other_versions__representative=F('pk')) |
+            Q(other_versions__representative__isnull=True)
+        ).order_by("start_time", "title__unaccent__lower")
+        
+        qs = qs.select_related("exact_location").select_related("category").select_related("other_versions").select_related("other_versions__representative")
+        self.events = qs
 
         firstdate = datetime.fromordinal(self.c_firstdate.toordinal())
         if firstdate.tzinfo is None or firstdate.tzinfo.utcoffset(firstdate) is None:
@@ -278,14 +315,14 @@ class CalendarList:
     def time_intervals_list_first(self):
         return self.time_intervals_list(True)
 
-    def export_to_ics(self):
+    def export_to_ics(self, request):
         from .models import Event
         events = [event for day in self.get_calendar_days().values() for event in day.events]
-        return Event.export_to_ics(events)
+        return Event.export_to_ics(events, request)
 
 
 class CalendarMonth(CalendarList):
-    def __init__(self, year, month, filter):
+    def __init__(self, year, month, filter, qs=None):
         self.year = year
         self.month = month
         r = calendar.monthrange(year, month)
@@ -293,7 +330,7 @@ class CalendarMonth(CalendarList):
         first = date(year, month, 1)
         last = date(year, month, r[1])
 
-        super().__init__(first, last, filter)
+        super().__init__(first, last, filter, qs)
 
     def get_month_name(self):
         return self.firstdate.strftime("%B")
@@ -306,14 +343,14 @@ class CalendarMonth(CalendarList):
 
 
 class CalendarWeek(CalendarList):
-    def __init__(self, year, week, filter):
+    def __init__(self, year, week, filter, qs=None):
         self.year = year
         self.week = week
 
         first = date.fromisocalendar(self.year, self.week, 1)
         last = date.fromisocalendar(self.year, self.week, 7)
 
-        super().__init__(first, last, filter)
+        super().__init__(first, last, filter, qs)
 
     def next_week(self):
         return self.firstdate + timedelta(days=7)
@@ -323,8 +360,8 @@ class CalendarWeek(CalendarList):
 
 
 class CalendarDay(CalendarList):
-    def __init__(self, date, filter=None):
-        super().__init__(date, date, filter, exact=True)
+    def __init__(self, date, filter=None, qs=None):
+        super().__init__(date, date, filter=filter, qs=qs, exact=True)
 
     def get_events(self):
         return self.calendar_days_list()[0].events

src/agenda_culturel/celery.py

@@ -6,7 +6,8 @@ from celery.schedules import crontab
 from celery.utils.log import get_task_logger
 from celery.exceptions import MaxRetriesExceededError
 import time as time_
-
+from django.conf import settings
+from celery.signals import worker_ready
 
 from contextlib import contextmanager
 
@@ -145,6 +146,10 @@ def run_recurrent_import_internal(rimport, downloader, req_id):
         extractor = fbevents.CExtractor()
     elif rimport.processor == RecurrentImport.PROCESSOR.C3C:
         extractor = c3c.CExtractor()
+    elif rimport.processor == RecurrentImport.PROCESSOR.ARACHNEE:
+        extractor = arachnee.CExtractor()
+    elif rimport.processor == RecurrentImport.PROCESSOR.LERIO:
+        extractor = lerio.CExtractor()
     else:
         extractor = None
 
@@ -160,13 +165,14 @@ def run_recurrent_import_internal(rimport, downloader, req_id):
     location = rimport.defaultLocation
     tags = rimport.defaultTags
     published = rimport.defaultPublished
+    organisers = [] if rimport.defaultOrganiser is None else [rimport.defaultOrganiser.pk]
 
     try:
         # get events from website
         events = u2e.process(
             url,
             browsable_url,
-            default_values={"category": category, "location": location, "tags": tags},
+            default_values={"category": category, "location": location, "tags": tags, "organisers": organisers},
             published=published,
         )
 
@@ -245,6 +251,23 @@ def daily_imports(self):
     run_recurrent_imports_from_list([imp.pk for imp in imports])
 
 
+SCREENSHOT_FILE = settings.MEDIA_ROOT + '/screenshot.png'
+
+@app.task(bind=True)
+def screenshot(self):
+    downloader = ChromiumHeadlessDownloader(noimage=False)
+    downloader.screenshot("https://pommesdelune.fr", SCREENSHOT_FILE)
+
+@worker_ready.connect
+def at_start(sender, **k):
+    if not os.path.isfile(SCREENSHOT_FILE):
+        logger.info("Init screenshot file")
+        with sender.app.connection() as conn:
+            sender.app.send_task('agenda_culturel.celery.screenshot', None, connection=conn)
+    else:
+        logger.info("Screenshot file already exists")
+
+
 @app.task(bind=True)
 def run_all_recurrent_imports(self):
     from agenda_culturel.models import RecurrentImport
@@ -286,7 +309,7 @@ def weekly_imports(self):
     run_recurrent_imports_from_list([imp.pk for imp in imports])
 
 @app.task(base=ChromiumTask, bind=True)
-def import_events_from_url(self, url, cat):
+def import_events_from_url(self, url, cat, tags, force=False, user_id=None):
     from .db_importer import DBImporterEvents
     from agenda_culturel.models import RecurrentImport, BatchImportation
     from agenda_culturel.models import Event, Category
@@ -302,9 +325,9 @@ def import_events_from_url(self, url, cat):
             url = Extractor.clean_url(url)
 
             # we check if the url is known
-            existing = Event.objects.filter(uuids__contains=[url])
+            existing = None if force else Event.objects.filter(uuids__contains=[url])
             # if it's unknown
-            if len(existing) == 0:
+            if force or len(existing) == 0:
 
                 # create an importer
                 importer = DBImporterEvents(self.request.id)
@@ -320,7 +343,7 @@ def import_events_from_url(self, url, cat):
                     # set default values
                     values = {}
                     if cat is not None:
-                        values = {"category": cat}
+                        values = {"category": cat, "tags": tags}
 
                     # get event
                     events = u2e.process(
@@ -332,7 +355,7 @@ def import_events_from_url(self, url, cat):
                         json_events = json.dumps(events, default=str)
 
                         # import events (from json)
-                        success, error_message = importer.import_events(json_events)
+                        success, error_message = importer.import_events(json_events, user_id)
 
                         # finally, close task
                         close_import_task(self.request.id, success, error_message, importer)
@@ -349,13 +372,14 @@ def import_events_from_url(self, url, cat):
 
 
 @app.task(base=ChromiumTask, bind=True)
-def import_events_from_urls(self, urls_and_cats):
-    for ucat in urls_and_cats:
+def import_events_from_urls(self, urls_cat_tags, user_id=None):
+    for ucat in urls_cat_tags:
         if ucat is not None:
             url = ucat[0]
             cat = ucat[1]
+            tags = ucat[2]
 
-            import_events_from_url.delay(url, cat)
+            import_events_from_url.delay(url, cat, tags, user_id=user_id)
 
 
 app.conf.beat_schedule = {
@@ -364,6 +388,10 @@ app.conf.beat_schedule = {
         # Daily imports at 3:14 a.m.
         "schedule": crontab(hour=3, minute=14),
     },
+    "daily_screenshot": {
+        "task": "agenda_culturel.celery.screenshot",
+        "schedule": crontab(hour=3, minute=3),
+    },
     "weekly_imports": {
         "task": "agenda_culturel.celery.weekly_imports",
         # Daily imports on Mondays at 2:22 a.m.

src/agenda_culturel/db_importer.py

@@ -11,6 +11,7 @@ class DBImporterEvents:
     def __init__(self, celery_id):
         self.celery_id = celery_id
         self.error_message = ""
+        self.user_id = None
         self.init_result_properties()
         self.today = timezone.now().date().isoformat()
 
@@ -34,9 +35,10 @@ class DBImporterEvents:
     def get_nb_removed_events(self):
         return self.nb_removed
 
-    def import_events(self, json_structure):
+    def import_events(self, json_structure, user_id=None):
         print(json_structure)
         self.init_result_properties()
+        self.user_id = user_id
 
         try:
             structure = json.loads(json_structure)
@@ -95,7 +97,7 @@ class DBImporterEvents:
 
     def save_imported(self):
         self.db_event_objects, self.nb_updated, self.nb_removed = Event.import_events(
-            self.event_objects, remove_missing_from_source=self.url
+            self.event_objects, remove_missing_from_source=self.url, user_id=self.user_id
         )
 
     def is_valid_event_structure(self, event):

src/agenda_culturel/filters.py

@@ -0,0 +1,505 @@
+import django_filters
+from django.utils.translation import gettext_lazy as _
+from django import forms
+from django.contrib.postgres.search import SearchQuery, SearchHeadline
+from django.db.models import Count, Q
+
+from django.http import QueryDict
+from django.contrib.gis.measure import D
+
+from django.forms import (
+    ModelForm,
+    ValidationError,
+    TextInput,
+    Form,
+    URLField,
+    MultipleHiddenInput,
+    Textarea,
+    CharField,
+    ChoiceField,
+    RadioSelect,
+    MultipleChoiceField,
+    BooleanField,
+    HiddenInput,
+    ModelChoiceField,
+)
+
+from .forms import (
+    URLSubmissionForm,
+    EventForm,
+    BatchImportationForm,
+    FixDuplicates,
+    SelectEventInList,
+    MergeDuplicates,
+    RecurrentImportForm,
+    CategorisationRuleImportForm,
+    CategorisationForm,
+    EventAddPlaceForm,
+    PlaceForm,
+)
+
+from .models import (
+    ReferenceLocation,
+    RecurrentImport,
+    Tag,
+    Event,
+    Category,
+    Message,
+    DuplicatedEvents
+)
+
+
+class EventFilter(django_filters.FilterSet):
+    RECURRENT_CHOICES = [
+        ("remove_recurrent", "Masquer les événements récurrents"),
+        ("only_recurrent", "Montrer uniquement les événements récurrents"),
+    ]
+
+    DISTANCE_CHOICES = [5, 10, 15, 30]
+
+    position = django_filters.ModelChoiceFilter(
+        label="À proximité de",
+        method="no_filter",
+        empty_label=_("Select a location"),
+        queryset=ReferenceLocation.objects.all().order_by("-main", "name__unaccent")
+    )
+
+    radius = django_filters.ChoiceFilter(
+        label="Dans un rayon de",
+        method="no_filter",
+        choices=[(x, str(x) + " km") for x in DISTANCE_CHOICES],
+        null_label=None,
+        empty_label=None
+    )
+
+    exclude_tags = django_filters.MultipleChoiceFilter(
+        label="Exclure les étiquettes",
+        choices=[],
+        lookup_expr="icontains",
+        field_name="tags",
+        exclude=True,
+        widget=forms.SelectMultiple,
+    )
+
+    tags = django_filters.MultipleChoiceFilter(
+        label="Inclure les étiquettes",
+        choices=[],
+        lookup_expr="icontains",
+        conjoined=True,
+        field_name="tags",
+        widget=forms.SelectMultiple,
+    )
+
+    recurrences = django_filters.ChoiceFilter(
+        label="Inclure la récurrence",
+        choices=RECURRENT_CHOICES,
+        method="filter_recurrences",
+    )
+
+    category = django_filters.ModelMultipleChoiceFilter(
+        label="Filtrer par catégories",
+        field_name="category__id",
+        to_field_name="id",
+        queryset=Category.objects.all(),
+        widget=MultipleHiddenInput,
+    )
+
+
+    status = django_filters.MultipleChoiceFilter(
+        label="Filtrer par status",
+        choices=Event.STATUS.choices,
+        field_name="status",
+        widget=forms.CheckboxSelectMultiple,
+    )
+
+    class Meta:
+        model = Event
+        fields = ["category", "tags", "exclude_tags", "status", "recurrences"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not kwargs["request"].user.is_authenticated:
+            self.form.fields.pop("status")
+        self.form.fields["exclude_tags"].choices = Tag.get_tag_groups(exclude=True, nb_suggestions=0)
+        self.form.fields["tags"].choices = Tag.get_tag_groups(include=True)
+
+    def filter_recurrences(self, queryset, name, value):
+        # construct the full lookup expression
+        lookup = "__".join([name, "isnull"])
+        return queryset.filter(**{lookup: value == "remove_recurrent"})
+
+    def no_filter(self, queryset, name, value): 
+        return queryset
+
+    @property
+    def qs(self):
+        parent = super().qs
+        if self.get_cleaned_data("position") is None or self.get_cleaned_data("radius") is None:
+            return parent
+        d = self.get_cleaned_data("radius")
+        p = self.get_cleaned_data("position")
+        if not isinstance(d, str) or not isinstance(p, ReferenceLocation):
+            return parent
+        p = p.location
+
+        return parent.exclude(exact_location=False).filter(exact_location__location__distance_lt=(p, D(km=d)))
+
+    def get_url(self):
+        if isinstance(self.form.data, QueryDict):
+            return self.form.data.urlencode()
+        else:
+            return ""
+
+    def get_full_url(self):
+        return self.request.get_full_path()
+
+    def get_url_remove_categories(self, catpks, full_path = None):
+        if full_path is None:
+            full_path = self.request.get_full_path()
+
+        result = full_path
+        for catpk in catpks:
+            result = result.replace('category=' + str(catpk), '')
+            result = result.replace('?&', '?')
+            result = result.replace('&&', '&')
+        return result
+
+    def get_url_add_categories(self, catpks, full_path = None):
+        if full_path is None:
+            full_path = self.request.get_full_path()
+
+        result = full_path
+        for catpk in catpks:
+            result = result + ('&' if '?' in full_path else '?') + 'category=' + str(catpk)
+        return result
+
+    def get_url_without_filters_only_cats(self):
+        return self.get_url_without_filters(True)
+
+
+    def get_url_without_filters(self, only_categories=False):
+
+        if only_categories:
+            # on repart d'une url sans option
+            result = self.request.get_full_path().split("?")[0]
+            # on ajoute toutes les catégories
+            result = self.get_url_add_categories([c.pk for c in self.get_categories()], result)
+        else:
+            # on supprime toutes les catégories
+            result = self.get_url_remove_categories([c.pk for c in self.get_categories()])
+
+        return result
+
+
+    def get_cleaned_data(self, name):
+
+        try:
+            return self.form.cleaned_data[name]
+        except AttributeError:
+            return {}
+        except KeyError:
+            return {}
+    
+    def get_categories(self):
+        return self.get_cleaned_data("category")
+    
+    def has_category(self):
+        return "category" in self.form.cleaned_data and len(self.get_cleaned_data("category")) > 0
+
+    def get_tags(self):
+        return self.get_cleaned_data("tags")
+
+    def get_exclude_tags(self):
+        return self.get_cleaned_data("exclude_tags")
+
+    def get_status(self):
+        return self.get_cleaned_data("status")
+
+    def get_position(self):
+        return self.get_cleaned_data("position")
+
+    def get_radius(self):
+        return self.get_cleaned_data("radius")
+
+    def to_str(self, prefix=''):
+        self.form.full_clean()
+        result = ' '.join([c.name for c in self.get_categories()] + [t for t in self.get_tags()] + ["~" + t for t in self.get_exclude_tags()]  + [str(self.get_position()), str(self.get_radius())])
+        if len(result) > 0:
+            result = prefix + result
+        return result
+
+    def get_status_names(self):
+        if "status" in self.form.cleaned_data:
+            return [
+                dict(Event.STATUS.choices)[s] for s in self.get_cleaned_data("status")
+            ]
+        else:
+            return []
+
+    def get_recurrence_filtering(self):
+        if "recurrences" in self.form.cleaned_data:
+            d = dict(self.RECURRENT_CHOICES)
+            v = self.form.cleaned_data["recurrences"]
+            if v in d:
+                return d[v]
+            else:
+                return ""
+        else:
+            return ""
+
+
+    def is_resetable(self, only_categories=False):
+        if only_categories:
+            return len(self.get_cleaned_data("category")) != 0
+        else:
+            if self.request.user.is_authenticated:
+                if (
+                    len(self.get_cleaned_data("status")) != 1
+                    or
+                    self.get_cleaned_data("status")[0] != Event.STATUS.PUBLISHED
+                ):
+                    return True
+            else:
+                if (
+                    len(self.get_cleaned_data("status")) != 0
+                ):
+                    return True
+            return (
+                len(self.get_cleaned_data("tags")) != 0
+                or len(self.get_cleaned_data("exclude_tags")) != 0
+                or len(self.get_cleaned_data("recurrences")) != 0
+                or ((not self.get_cleaned_data("position") is None) and (not self.get_cleaned_data("radius") is None))
+            )
+
+    def is_active(self, only_categories=False):
+        if only_categories:
+            return len(self.get_cleaned_data("category")) != 0
+        else:
+            return (
+                len(self.get_cleaned_data("status")) != 0
+                or len(self.get_cleaned_data("tags")) != 0
+                or len(self.get_cleaned_data("exclude_tags")) != 0
+                or len(self.get_cleaned_data("recurrences")) != 0
+                or ((not self.get_cleaned_data("position") is None) and (not self.get_cleaned_data("radius") is None))
+            )
+
+    def is_selected(self, cat):
+        return "category" in self.form.cleaned_data and cat in self.form.cleaned_data["category"]
+
+    def is_selected_tag(self, tag):
+        return "tags" in self.form.cleaned_data and tag in self.form.cleaned_data["tags"]
+
+    def get_url_add_tag(self, tag):
+        full_path = self.request.get_full_path()
+
+        result = full_path + ('&' if '?' in full_path else '?') + 'tags=' + str(tag)
+
+        return result
+
+    def tag_exists(self, tag):
+        return tag in [t[0] for g in self.form.fields["tags"].choices for t in g[1]]
+
+    def set_default_values(request):
+        if request.user.is_authenticated:
+            if request.GET.get('status', None) == None:
+                tempdict = request.GET.copy()
+                tempdict['status'] = 'published'
+                request.GET = tempdict
+                return request
+        return request
+
+    def get_position_radius(self):
+        if self.get_cleaned_data("position") is None or self.get_cleaned_data("radius") is None:
+            return ""
+        else:
+            return str(self.get_cleaned_data("position")) + ' (' + str(self.get_cleaned_data("radius")) + ' km)'
+
+    def is_filtered_by_position_radius(self):
+        return not self.get_cleaned_data("position") is None and not self.get_cleaned_data("radius") is None
+
+    def get_url_add_suggested_position(self, location):
+        result = self.request.get_full_path()
+        return result + ('&' if '?' in result else '?') + 'position=' + str(location.pk) + "&radius=" + str(location.suggested_distance)
+
+
+class EventFilterAdmin(django_filters.FilterSet):
+    status = django_filters.MultipleChoiceFilter(
+        choices=Event.STATUS.choices, widget=forms.CheckboxSelectMultiple
+    )
+
+    representative = django_filters.MultipleChoiceFilter(
+        label=_("Representative version"),
+        choices=[(True, _("Yes")), (False, _("Non"))], 
+        method="filter_by_representative",
+        widget=forms.CheckboxSelectMultiple)
+
+    import_sources = django_filters.ModelChoiceFilter(
+        label=_("Imported from"),
+        method="filter_by_source",
+        queryset=RecurrentImport.objects.all().order_by("name__unaccent")
+    )
+
+    def filter_by_source(self, queryset, name, value):
+        src = RecurrentImport.objects.get(pk=value.pk).source
+        return queryset.filter(import_sources__contains=[src])
+
+    def filter_by_representative(self, queryset, name, value):
+        if value is None or len(value) != 1:
+            return queryset
+        else:
+            q = (Q(other_versions__isnull=True) |
+                 Q(other_versions__representative=F('pk')) |
+                 Q(other_versions__representative__isnull=True))
+            if value[0] == True:
+                return queryset.filter(q)
+            else:
+                return queryset.exclude(q)
+
+
+    class Meta:
+        model = Event
+        fields = ["status"]
+
+
+class MessagesFilterAdmin(django_filters.FilterSet):
+    closed = django_filters.MultipleChoiceFilter(
+        label="Status",
+        choices=((True, _("Closed")), (False, _("Open"))),
+        widget=forms.CheckboxSelectMultiple,
+    )
+    spam = django_filters.MultipleChoiceFilter(
+        label="Spam",
+        choices=((True, _("Spam")), (False, _("Non spam"))),
+        widget=forms.CheckboxSelectMultiple,
+    )
+
+    class Meta:
+        model = Message
+        fields = ["closed", "spam"]
+
+
+class SimpleSearchEventFilter(django_filters.FilterSet):
+    q = django_filters.CharFilter(method="custom_filter", 
+        label=_("Search"),
+        widget=forms.TextInput(attrs={"type": "search"})
+    )
+
+    status = django_filters.MultipleChoiceFilter(
+        label="Filtrer par status",
+        choices=Event.STATUS.choices,
+        field_name="status",
+        widget=forms.CheckboxSelectMultiple,
+    )
+
+    def custom_filter(self, queryset, name, value):
+        search_query = SearchQuery(value, config="french")
+        qs = queryset.filter(
+            Q(title__icontains=value)
+            | Q(category__name__icontains=value)
+            | Q(tags__icontains=[value])
+            | Q(exact_location__name__icontains=value)
+            | Q(description__icontains=value)
+        )
+        for f in ["title", "category__name", "exact_location__name", "description"]:
+            params = {
+                f
+                + "_hl": SearchHeadline(
+                    f,
+                    search_query,
+                    start_sel='<span class="highlight">',
+                    stop_sel="</span>",
+                    config="french",
+                )
+            }
+            qs = qs.annotate(**params)
+        return qs
+
+    class Meta:
+        model = Event
+        fields = ["q"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not kwargs["request"].user.is_authenticated:
+            self.form.fields.pop("status")
+
+
+class SearchEventFilter(django_filters.FilterSet):
+    tags = django_filters.CharFilter(lookup_expr="icontains")
+    title = django_filters.CharFilter(method="hl_filter_contains")
+    location = django_filters.CharFilter(method="hl_filter_contains")
+    description = django_filters.CharFilter(method="hl_filter_contains")
+    start_day = django_filters.DateFromToRangeFilter(
+        widget=django_filters.widgets.RangeWidget(attrs={"type": "date"})
+    )
+    status = django_filters.MultipleChoiceFilter(
+        label="Filtrer par status",
+        choices=Event.STATUS.choices,
+        field_name="status",
+        widget=forms.CheckboxSelectMultiple,
+    )
+
+    o = django_filters.OrderingFilter(
+        # tuple-mapping retains order
+        fields=(
+            ("title", "title"),
+            ("description", "description"),
+            ("start_day", "start_day"),
+        ),
+    )
+
+    def hl_filter_contains(self, queryset, name, value):
+        # first check if it contains
+        filter_contains = {name + "__contains": value}
+        queryset = queryset.filter(**filter_contains)
+
+        # then hightlight the result
+        search_query = SearchQuery(value, config="french")
+        params = {
+            name
+            + "_hl": SearchHeadline(
+                name,
+                search_query,
+                start_sel='<span class="highlight">',
+                stop_sel="</span>",
+                config="french",
+            )
+        }
+        return queryset.annotate(**params)
+
+    class Meta:
+        model = Event
+        fields = ["title", "location", "description", "category", "tags", "start_day"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not kwargs["request"].user.is_authenticated:
+            self.form.fields.pop("status")
+
+
+class DuplicatedEventsFilter(django_filters.FilterSet):
+    fixed = django_filters.BooleanFilter(
+        label="Résolu",
+        field_name='representative', method="fixed_qs")
+
+    class Meta:
+        model = DuplicatedEvents
+        fields = []
+
+    
+    def fixed_qs(self, queryset, name, value):
+        return DuplicatedEvents.not_fixed_qs(queryset, value)
+
+
+class RecurrentImportFilter(django_filters.FilterSet):
+
+    name = django_filters.ModelMultipleChoiceFilter(
+        label="Filtrer par nom",
+        field_name="name",
+        queryset=RecurrentImport.objects.all().order_by("name__unaccent")
+    )
+
+    class Meta:
+        model = RecurrentImport
+        fields = ["name"]
+

src/agenda_culturel/forms.py

@@ -16,38 +16,136 @@ from django.forms import (
 )
 from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget
 
+from .utils import PlaceGuesser
 from .models import (
     Event,
     RecurrentImport,
     CategorisationRule,
-    ModerationAnswer,
-    ModerationQuestion,
     Place,
     Category,
+    Tag,
+    Message
 )
+from django.conf import settings
+from django.core.files import File
+
 from django.utils.translation import gettext_lazy as _
 from string import ascii_uppercase as auc
 from .templatetags.utils_extra import int_to_abc
 from django.utils.safestring import mark_safe
-from django.utils.timezone import localtime
 from django.utils.formats import localize
 from .templatetags.event_extra import event_field_verbose_name, field_to_html
+import os
 
 import logging
 
 logger = logging.getLogger(__name__)
 
+class GroupFormMixin:
+
+    template_name = 'agenda_culturel/forms/div_group.html'
+
+    class FieldGroup:
+
+        def __init__(self, id, label, display_label=False, maskable=False, default_masked=True):
+            self.id = id
+            self.label = label
+            self.display_label = display_label
+            self.maskable = maskable
+            self.default_masked = default_masked
+
+        def toggle_field_name(self):
+            return 'group_' + self.id
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.groups = []
+
+    def add_group(self, *args, **kwargs):
+        self.groups.append(GroupFormMixin.FieldGroup(*args, **kwargs))
+        if self.groups[-1].maskable:
+            self.fields[self.groups[-1].toggle_field_name()] = BooleanField(required=False)
+            self.fields[self.groups[-1].toggle_field_name()].toggle_group = True
+
+    def get_fields_in_group(self, g):
+        return [f for f in self.visible_fields() if not hasattr(f.field, "toggle_group") and hasattr(f.field, "group_id") and f.field.group_id == g.id]
+
+    def get_no_group_fields(self):
+        return [f for f in self.visible_fields() if not hasattr(f.field, "toggle_group") and (not hasattr(f.field, "group_id") or f.field.group_id == None)]
+
+    def fields_by_group(self):
+        return [(g, self.get_fields_in_group(g)) for g in self.groups] + [(GroupFormMixin.FieldGroup("other", _("Other")), self.get_no_group_fields())]
+
+    def clean(self):
+        result = super().clean()
+
+        if result:
+            data = dict(self.data)
+            # for each masked group, we remove data
+            for g in self.groups:
+                if g.maskable and not g.toggle_field_name() in data:
+                    fields = self.get_fields_in_group(g)
+                    for f in fields:
+                        self.cleaned_data[f.name] = None
+        
+        return result
+
+class TagForm(ModelForm):
+    required_css_class = 'required'
+
+    class Meta:
+        model = Tag
+        fields = ["name", "description", "in_included_suggestions", "in_excluded_suggestions", "principal"]
+        widgets = {
+            "name": HiddenInput()
+        }
+
+class TagRenameForm(Form):
+    required_css_class = 'required'
+
+    name = CharField(
+        label=_('Name of new tag'),
+        required=True
+    )
+
+    force = BooleanField(
+        label=_('Force renaming despite the existence of events already using the chosen tag.'),
+    )
+
+    def __init__(self, *args, **kwargs):
+        force = kwargs.pop("force", False)
+        name = kwargs.pop("name", None)
+        super().__init__(*args, **kwargs)
+        if not (force or (not len(args) == 0 and 'force' in args[0])):
+            del self.fields["force"]
+        if not name is None and self.fields["name"].initial is None:
+            self.fields["name"].initial = name
+    
+
+    def is_force(self):
+        return "force" in self.fields and self.cleaned_data["force"] == True
 
 class URLSubmissionForm(Form):
+    required_css_class = 'required'
+
     url = URLField(max_length=512)
     category = ModelChoiceField(
         label=_("Category"),
         queryset=Category.objects.all().order_by("name"),
-        initial=Category.get_default_category(),
-        empty_label=None,
-        help_text=_('Optional. If you don''t specify a category, we''ll find it for you.'),
+        initial=None,
         required=False,
     )
+    tags = MultipleChoiceField(
+        label=_("Tags"),
+        initial=None,
+        choices=[],
+        required=False
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["tags"].choices = Tag.get_tag_groups(all=True)
+
 
 
 
@@ -60,28 +158,58 @@ class DynamicArrayWidgetTags(DynamicArrayWidget):
 
 
 class RecurrentImportForm(ModelForm):
+    required_css_class = 'required'
+
+    defaultTags = MultipleChoiceField(
+        label=_("Tags"),
+        initial=None,
+        choices=[],
+        required=False
+    )
+
     class Meta:
         model = RecurrentImport
         fields = "__all__"
-        widgets = {
-            "defaultTags": DynamicArrayWidgetTags(),
-        }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["defaultTags"].choices = Tag.get_tag_groups(all=True)
 
 
 class CategorisationRuleImportForm(ModelForm):
+    required_css_class = 'required'
+
     class Meta:
         model = CategorisationRule
         fields = "__all__"
 
 
-class EventForm(ModelForm):
+class EventForm(GroupFormMixin, ModelForm):
+    required_css_class = 'required'
+
+    old_local_image = CharField(widget=HiddenInput(), required=False)
+    simple_cloning = CharField(widget=HiddenInput(), required=False)
+
+    tags = MultipleChoiceField(
+        label=_("Tags"),
+        initial=None,
+        choices=[],
+        required=False
+    )
+
+
     class Meta:
         model = Event
         exclude = [
-            "possibly_duplicated",
             "imported_date",
             "modified_date",
             "moderated_date",
+            "import_sources",
+            "image",
+            "moderated_by_user",
+            "modified_by_user",
+            "created_by_user",
+            "imported_by_user"
         ]
         widgets = {
             "start_day": TextInput(
@@ -100,20 +228,75 @@ class EventForm(ModelForm):
             ),
             "end_day": TextInput(attrs={"type": "date"}),
             "end_time": TextInput(attrs={"type": "time"}),
+            "other_versions": HiddenInput(),
             "uuids": MultipleHiddenInput(),
-            "import_sources": MultipleHiddenInput(),
             "reference_urls": DynamicArrayWidgetURLs(),
-            "tags": DynamicArrayWidgetTags(),
         }
 
     def __init__(self, *args, **kwargs):
         is_authenticated = kwargs.pop("is_authenticated", False)
+        self.cloning = kwargs.pop("is_cloning", False)
+        self.simple_cloning = kwargs.pop("is_simple_cloning", False)
         super().__init__(*args, **kwargs)
         if not is_authenticated:
             del self.fields["status"]
+            del self.fields["organisers"]
         self.fields['category'].queryset = self.fields['category'].queryset.order_by('name')
         self.fields['category'].empty_label = None
         self.fields['category'].initial = Category.get_default_category()
+        self.fields['tags'].choices = Tag.get_tag_groups(all=True)
+
+        # set groups
+        self.add_group('main', _('Main fields'))
+        self.fields['title'].group_id = 'main'
+
+        self.add_group('start', _('Start of event'))
+        self.fields['start_day'].group_id = 'start'
+        self.fields['start_time'].group_id = 'start'
+
+        self.add_group('end', _('End of event'))
+        self.fields['end_day'].group_id = 'end'
+        self.fields['end_time'].group_id = 'end'
+
+        self.add_group('recurrences', 
+            _('This is a recurring event'), 
+            maskable=True, 
+            default_masked=not (self.instance and 
+            self.instance.recurrences and
+            self.instance.recurrences.rrules and
+            len(self.instance.recurrences.rrules) > 0))
+
+        self.fields['recurrences'].group_id = 'recurrences'
+
+        self.add_group('details', _('Details'))
+        self.fields['description'].group_id = 'details'
+        if is_authenticated:
+            self.fields['organisers'].group_id = 'details'
+
+        self.add_group('location', _('Location'))
+        self.fields['location'].group_id = 'location'
+        self.fields['exact_location'].group_id = 'location'
+
+        self.add_group('illustration', _('Illustration'))
+        self.fields['local_image'].group_id = 'illustration'
+        self.fields['image_alt'].group_id = 'illustration'
+
+
+        if is_authenticated:
+            self.add_group('meta-admin', _('Meta information'))
+            self.fields['category'].group_id = 'meta-admin'
+            self.fields['tags'].group_id = 'meta-admin'
+            self.fields['status'].group_id = 'meta-admin'
+        else:
+            self.add_group('meta', _('Meta information'))
+            self.fields['category'].group_id = 'meta'
+            self.fields['tags'].group_id = 'meta'
+
+    def is_clone_from_url(self):
+        return self.cloning
+    
+    def is_simple_clone_from_url(self):
+        return self.simple_cloning
 
     def clean_end_day(self):
         start_day = self.cleaned_data.get("start_day")
@@ -141,8 +324,78 @@ class EventForm(ModelForm):
 
         return end_time
 
+    def clean(self):
+        super().clean()
+
+        # when cloning an existing event, we need to copy the local image
+        if self.cleaned_data['local_image'] is None and \
+                not self.cleaned_data['old_local_image'] is None and \
+                self.cleaned_data['old_local_image'] != "":
+            basename = self.cleaned_data['old_local_image']
+            old = settings.MEDIA_ROOT + "/" + basename
+            if os.path.isfile(old):
+                self.cleaned_data['local_image'] = File(name=basename, file=open(old, "rb"))
+
+
+class MultipleChoiceFieldAcceptAll(MultipleChoiceField):
+    def validate(self, value):
+        pass
+
+
+class EventModerateForm(ModelForm):
+    required_css_class = 'required'
+
+    tags = MultipleChoiceField(
+        label=_("Tags"),
+        help_text=_('Select tags from existing ones.'),
+        required=False
+    )
+
+    new_tags = MultipleChoiceFieldAcceptAll(
+        label=_("New tags"),
+        help_text=_('Create new labels (sparingly). Note: by starting your tag with the characters “TW:”, you''ll create a “trigger warning” tag, and the associated events will be announced as such.'),
+        widget=DynamicArrayWidget(),
+        required=False
+    )
+
+    class Meta:
+        model = Event
+        fields = [
+            "status",
+            "category",
+            "organisers",
+            "exact_location",
+            "tags"
+        ]
+        widgets = {
+            "status": RadioSelect
+        }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields['category'].queryset = self.fields['category'].queryset.order_by('name')
+        self.fields['category'].empty_label = None
+        self.fields['category'].initial = Category.get_default_category()
+        self.fields['tags'].choices = Tag.get_tag_groups(all=True)
+
+    def clean_new_tags(self):
+        return list(set(self.cleaned_data.get("new_tags")))
+
+    def clean(self):
+        super().clean()
+
+        if self.cleaned_data['tags'] is None:
+            self.cleaned_data['tags'] = []
+
+        if not self.cleaned_data.get('new_tags') is None:
+            self.cleaned_data['tags'] += self.cleaned_data.get('new_tags')
+        
+        self.cleaned_data['tags'] = list(set(self.cleaned_data['tags']))
+
 
 class BatchImportationForm(Form):
+    required_css_class = 'required'
+
     json = CharField(
         label="JSON",
         widget=Textarea(attrs={"rows": "10"}),
@@ -152,54 +405,64 @@ class BatchImportationForm(Form):
 
 
 class FixDuplicates(Form):
+    required_css_class = 'required'
+
     action = ChoiceField()
 
     def __init__(self, *args, **kwargs):
-        nb_events = kwargs.pop("nb_events", None)
+        edup = kwargs.pop("edup", None)
+        events = edup.get_duplicated()
+        nb_events = len(events)
         super().__init__(*args, **kwargs)
 
-        if nb_events == 2:
-            choices = [("NotDuplicates", "Ces événements sont différents")]
-            choices += [
-                (
-                    "SelectA",
-                    "Ces événements sont identiques, on garde A et on met B à la corbeille",
-                )
-            ]
-            choices += [
-                (
-                    "SelectB",
-                    "Ces événements sont identiques, on garde B et on met A à la corbeille",
-                )
-            ]
-            choices += [
-                ("Merge", "Ces événements sont identiques, on fusionne à la main")
-            ]
-        else:
-            choices = [("NotDuplicates", "Ces événements sont tous différents")]
-            for i in auc[0:nb_events]:
+        choices = []
+        initial = None
+        for i, e in enumerate(events):
+            if e.status != Event.STATUS.TRASH or e.modified():
+                msg = ""
+                if e.local_version():
+                    msg = _(" (locally modified version)")
+                    if e.status != Event.STATUS.TRASH:
+                        initial = "Select-" + str(e.pk)
+                if e.pure_import():
+                    msg = _(" (synchronized on import version)")
                 choices += [
                     (
-                        "Remove" + i,
-                        "L'événement "
-                        + i
-                        + " n'est pas identique aux autres, on le rend indépendant",
+                        "Select-" + str(e.pk),
+                        _("Select {} as representative version.").format(auc[i] + msg)
                     )
                 ]
-            for i in auc[0:nb_events]:
+
+        for i, e in enumerate(events):
+            if e.status != Event.STATUS.TRASH and e.local_version():
                 choices += [
                     (
-                        "Select" + i,
-                        "Ces événements sont identiques, on garde "
-                        + i
-                        + " et on met les autres à la corbeille",
+                        "Update-" + str(e.pk),
+                        _("Update {} using some fields from other versions (interactive mode).").format(auc[i])
                     )
                 ]
-            choices += [
-                ("Merge", "Ces événements sont identiques, on fusionne à la main")
-            ]
+
+
+        extra = ""
+        if edup.has_local_version():
+            extra = _(" Warning: a version is already locally modified.")
+
+        if initial is None:
+            initial = "Merge"
+        choices += [
+            ("Merge", _("Create a new version by merging (interactive mode).") + extra)
+        ]
+        for i, e in enumerate(events):
+            if e.status != Event.STATUS.TRASH:
+                choices += [
+                    (
+                        "Remove-" + str(e.pk),
+                        _("Make {} independent.").format(auc[i]))
+                ]
+        choices += [("NotDuplicates", _("Make all versions independent."))]
 
         self.fields["action"].choices = choices
+        self.fields["action"].initial = initial
 
     def is_action_no_duplicates(self):
         return self.cleaned_data["action"] == "NotDuplicates"
@@ -207,50 +470,62 @@ class FixDuplicates(Form):
     def is_action_select(self):
         return self.cleaned_data["action"].startswith("Select")
 
+    def is_action_update(self):
+        return self.cleaned_data["action"].startswith("Update")
+
     def is_action_remove(self):
         return self.cleaned_data["action"].startswith("Remove")
 
     def get_selected_event_code(self):
-        if self.is_action_select() or self.is_action_remove():
-            return self.cleaned_data["action"][-1]
+        if self.is_action_select() or self.is_action_remove() or self.is_action_update():
+            return int(self.cleaned_data["action"].split("-")[-1])
         else:
             return None
 
-    def get_selected_event_id(self):
-        selected = self.get_selected_event_code()
-        if selected is None:
-            return None
-        else:
-            return auc.rfind(selected)
-
     def get_selected_event(self, edup):
-        selected = self.get_selected_event_id()
-        return edup.get_duplicated()[selected]
+        selected = self.get_selected_event_code()
+        for e in edup.get_duplicated():
+            if e.pk == selected:
+                return e
+        return None
 
 
 class SelectEventInList(Form):
-    event = ChoiceField()
+    required_css_class = 'required'
+
+    event = ChoiceField(label=_('Event'))
 
     def __init__(self, *args, **kwargs):
         events = kwargs.pop("events", None)
         super().__init__(*args, **kwargs)
 
         self.fields["event"].choices = [
-            (e.pk, str(e.start_day) + " " + e.title + ", " + e.location) for e in events
+            (e.pk, str(e.start_day) + " " + e.title + ((", " + e.location) if e.location else "")) for e in events
         ]
 
 
 class MergeDuplicates(Form):
-    checkboxes_fields = ["reference_urls", "description"]
+    required_css_class = 'required'
+
+    checkboxes_fields = ["reference_urls", "description", "tags"]
 
     def __init__(self, *args, **kwargs):
         self.duplicates = kwargs.pop("duplicates", None)
-        nb_events = self.duplicates.nb_duplicated()
+        self.event = kwargs.pop("event", None)
+        self.events = list(self.duplicates.get_duplicated())
+        nb_events = len(self.events)
         super().__init__(*args, **kwargs)
 
-        choices = [
-            ("event" + i, "Valeur de l'évenement " + i) for i in auc[0:nb_events]
-        ]
+        
+        if self.event:
+            choices = [("event_" + str(self.event.pk), _("Value of the selected version"))] + \
+            [
+                ("event_" + str(e.pk), _("Value of version {}").format(e.pk)) for e in self.events if e != self.event
+            ]
+        else:
+            choices = [
+                ("event_" + str(e.pk), _("Value of version {}").format(e.pk)) for e in self.events
+            ]
 
         for f in self.duplicates.get_items_comparison():
             if not f["similar"]:
@@ -265,7 +540,7 @@ class MergeDuplicates(Form):
 
     def as_grid(self):
         result = '<div class="grid">'
-        for i, e in enumerate(self.duplicates.get_duplicated()):
+        for i, e in enumerate(self.events):
             result += '<div class="grid entete-badge">'
             result += '<div class="badge-large">' + int_to_abc(i) + "</div>"
             result += "<ul>"
@@ -273,17 +548,17 @@ class MergeDuplicates(Form):
                 '<li><a href="' + e.get_absolute_url() + '">' + e.title + "</a></li>"
             )
             result += (
-                "<li>Création&nbsp;: " + localize(localtime(e.created_date)) + "</li>"
+                "<li>Création&nbsp;: " + localize(e.created_date) + "</li>"
             )
             result += (
                 "<li>Dernière modification&nbsp;: "
-                + localize(localtime(e.modified_date))
+                + localize(e.modified_date)
                 + "</li>"
             )
             if e.imported_date:
                 result += (
                     "<li>Dernière importation&nbsp;: "
-                    + localize(localtime(e.imported_date))
+                    + localize(e.imported_date)
                     + "</li>"
                 )
             result += "</ul>"
@@ -301,93 +576,87 @@ class MergeDuplicates(Form):
                 )
             else:
                 result += "<fieldset>"
+                if key in self.errors:
+                    result += '<div class="message error"><ul>'
+                    for err in self.errors[key]:
+                        result += "<li>" + err + "</li>"
+                    result += "</ul></div>"
                 result += '<div class="grid comparison-item">'
                 if hasattr(self, "cleaned_data"):
                     checked = self.cleaned_data.get(key)
                 else:
                     checked = self.fields[key].initial
 
-                for i, (v, radio) in enumerate(
-                    zip(e["values"], self.fields[e["key"]].choices)
-                ):
-                    result += '<div class="duplicated">'
-                    id = "id_" + key + "_" + str(i)
-                    value = "event" + auc[i]
-
-                    result += '<input id="' + id + '" name="' + key + '"'
-                    if key in MergeDuplicates.checkboxes_fields:
-                        result += ' type="checkbox"'
-                        if value in checked:
-                            result += " checked"
-                    else:
-                        result += ' type="radio"'
-                        if checked == value:
-                            result += " checked"
-                    result += ' value="' + value + '"'
-                    result += ">"
-                    result += (
-                        '<div class="badge-small">'
-                        + int_to_abc(i)
-                        + "</div>"
-                        + str(field_to_html(v, e["key"]))
-                        + "</div>"
-                    )
+                i = 0
+                if self.event:
+                    idx = self.events.index(self.event)
+                    result += self.comparison_item(key, i, e["values"][idx], self.fields[e["key"]].choices[idx], self.event, checked)
+                    i += 1
+                
+                for (v, radio, ev) in zip(e["values"], self.fields[e["key"]].choices, self.events):
+                    if self.event is None or ev != self.event:
+                        result += self.comparison_item(key, i, v, radio, ev, checked)
+                        i += 1
                 result += "</div></fieldset>"
 
         return mark_safe(result)
 
-    def get_selected_events_id(self, key):
+    def comparison_item(self, key, i, v, radio, ev, checked):
+        result = '<div class="duplicated">'
+        id = "id_" + key + "_" + str(ev.pk)
+        value = "event_" + str(ev.pk)
+
+        result += '<input id="' + id + '" name="' + key + '"'
+        if key in MergeDuplicates.checkboxes_fields:
+            result += ' type="checkbox"'
+            if checked and value in checked:
+                result += " checked"
+        else:
+            result += ' type="radio"'
+            if checked == value:
+                result += " checked"
+        result += ' value="' + value + '"'
+        result += ">"
+        result += (
+            '<div class="badge-small">'
+            + int_to_abc(i)
+            + "</div>")
+        result += "<div>"
+        if key == "image":
+            result += str(field_to_html(ev.local_image, "local_image")) + "</div>"
+            result += "<div>Lien d'import&nbsp;: "
+
+        result += (str(field_to_html(v, key)) + "</div>")
+        result += "</div>"
+        return result
+
+
+    def get_selected_events(self, key):
         value = self.cleaned_data.get(key)
         if key not in self.fields:
             return None
         else:
             if isinstance(value, list):
-                return [auc.rfind(v[-1]) for v in value]
+                selected = [int(v.split("_")[-1]) for v in value]
+                result = []
+                for s in selected:
+                    for e in self.duplicates.get_duplicated():
+                        if e.pk == s:
+                            result.append(e)
+                            break
+                return result    
             else:
-                return auc.rfind(value[-1])
+                selected = int(value.split("_")[-1])
+                for e in self.duplicates.get_duplicated():
+                    if e.pk == selected:
+                        return e
 
-
-class ModerationQuestionForm(ModelForm):
-    class Meta:
-        model = ModerationQuestion
-        fields = "__all__"
-
-
-class ModerationAnswerForm(ModelForm):
-    class Meta:
-        model = ModerationAnswer
-        exclude = ["question"]
-        widgets = {
-            "adds_tags": DynamicArrayWidgetTags(),
-            "removes_tags": DynamicArrayWidgetTags(),
-        }
-
-
-class ModerateForm(ModelForm):
-    class Meta:
-        model = Event
-        fields = []
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        mqs = ModerationQuestion.objects.all()
-        mas = ModerationAnswer.objects.all()
-
-        for q in mqs:
-            self.fields[q.complete_id()] = ChoiceField(
-                widget=RadioSelect,
-                label=q.question,
-                choices=[(a.pk, a.html_description()) for a in mas if a.question == q],
-                required=True,
-            )
-            for a in mas:
-                if a.question == q and a.valid_event(self.instance):
-                    self.fields[q.complete_id()].initial = a.pk
-                    break
+                return None
 
 
 class CategorisationForm(Form):
+    required_css_class = 'required'
+
     def __init__(self, *args, **kwargs):
         if "events" in kwargs:
             events = kwargs.pop("events", None)
@@ -418,6 +687,8 @@ class CategorisationForm(Form):
 
 
 class EventAddPlaceForm(Form):
+    required_css_class = 'required'
+
     place = ModelChoiceField(
         label=_("Place"),
         queryset=Place.objects.all().order_by("name"),
@@ -443,18 +714,20 @@ class EventAddPlaceForm(Form):
         if self.cleaned_data.get("place"):
             place = self.cleaned_data.get("place")
             self.instance.exact_location = place
-            self.instance.save()
+            self.instance.save(update_fields=["exact_location"])
             if self.cleaned_data.get("add_alias"):
                 if place.aliases:
-                    place.aliases.append(self.instance.location)
+                    place.aliases.append(self.instance.location.strip())
                 else:
-                    place.aliases = [self.instance.location]
+                    place.aliases = [self.instance.location.strip()]
                 place.save()
 
         return self.instance
 
 
-class PlaceForm(ModelForm):
+class PlaceForm(GroupFormMixin, ModelForm):
+    required_css_class = 'required'
+
     apply_to_all = BooleanField(
         initial=True,
         label=_(
@@ -468,13 +741,70 @@ class PlaceForm(ModelForm):
         fields = "__all__"
         widgets = {"location": TextInput()}
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.add_group('header', _('Header'))
+        self.fields['name'].group_id = 'header'
+        
+        
+        self.add_group('address', _('Address'))
+        self.fields['address'].group_id = 'address'
+        self.fields['postcode'].group_id = 'address'
+        self.fields['city'].group_id = 'address'
+        self.fields['location'].group_id = 'address'
+
+        self.add_group('meta', _('Meta'))
+        self.fields['aliases'].group_id = 'meta'
+
+        self.add_group('information', _('Information'))
+        self.fields['description'].group_id = 'information'
+
     def as_grid(self):
-        return mark_safe(
-            '<div class="grid"><div>'
+        result = ('<div class="grid"><div>'
             + super().as_p()
-            + '</div><div><div class="map-widget">'
-            + '<div id="map_location" style="width: 100%; aspect-ratio: 16/9"></div><p>Cliquez pour ajuster la position GPS</p></div></div></div>'
-        )
+            + '''</div><div><div class="map-widget">
+            <div id="map_location" style="width: 100%; aspect-ratio: 16/9"></div>
+            <p>Cliquez pour ajuster la position GPS</p></div>
+            <input type="checkbox" role="switch" id="lock_position">Verrouiller la position</lock>
+            <script>
+                document.getElementById("lock_position").onclick = function() {
+                    const field = document.getElementById("id_location");
+                    if (this.checked)
+                        field.setAttribute("readonly", true);
+                    else
+                        field.removeAttribute("readonly");
+                }
+            </script>
+            </div></div>''')
+        
+        return mark_safe(result)
 
     def apply(self):
         return self.cleaned_data.get("apply_to_all")
+
+class MessageForm(ModelForm):
+
+    class Meta:
+        model = Message
+        fields = ["subject", "name", "email", "message", "related_event"]
+        widgets = {"related_event": HiddenInput(), "user": HiddenInput() }
+
+    def __init__(self, *args, **kwargs):
+        self.event = kwargs.pop("event", False)
+        self.internal = kwargs.pop("internal", False)
+        super().__init__(*args, **kwargs)
+        self.fields['related_event'].required = False
+        if self.internal:
+            self.fields.pop("name")
+            self.fields.pop("email")
+
+class MessageEventForm(ModelForm):
+
+    class Meta:
+        model = Message
+        fields = ["message"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["message"].label = _("Add a comment")

src/agenda_culturel/import_tasks/custom_extractors/arachnee.py

@@ -0,0 +1,113 @@
+from ..generic_extractors import *
+from bs4 import BeautifulSoup
+
+# A class dedicated to get events from Arachnée Concert
+# URL: https://www.arachnee-concerts.com/agenda-des-concerts/
+class CExtractor(TwoStepsExtractorNoPause):
+
+    def __init__(self):
+        super().__init__()
+        self.possible_dates = {}
+        self.theater = None
+
+    def extract(
+        self,
+        content,
+        url,
+        url_human=None,
+        default_values=None,
+        published=False,
+        only_future=True,
+        ignore_404=True
+    ):
+        match = re.match(r".*\&theatres=([^&]*)&.*", url)
+        if match:
+            self.theater = match[1]
+
+        return super().extract(content, url, url_human, default_values, published, only_future, ignore_404)
+
+    def build_event_url_list(self, content, infuture_days=180):
+        
+        soup = BeautifulSoup(content, "html.parser")
+
+        containers = soup.select("ul.event_container>li")
+        if containers:
+            for c in containers:
+                d = Extractor.parse_french_date(c.select_one(".date").text)
+                l = c.select_one(".event_auditory").text
+                if (self.theater is None or (l.startswith(self.theater))) and d < datetime.date.today() + timedelta(days=infuture_days):
+                    t = Extractor.parse_french_time(c.select_one(".time").text)
+                    e_url = c.select_one(".info a")["href"]
+                    if not e_url in self.possible_dates:
+                        self.possible_dates[e_url] = []
+                    self.possible_dates[e_url].append((str(d) + " " + str(t)))
+                    self.add_event_url(e_url)
+
+    def add_event_from_content(
+        self,
+        event_content,
+        event_url,
+        url_human=None,
+        default_values=None,
+        published=False,
+    ):
+
+        soup = BeautifulSoup(event_content, "html.parser")
+        title = ", ".join([x.text for x in [soup.select_one(y) for y in [".page_title", ".artiste-subtitle"]] if x])
+        
+        image = soup.select_one(".entry-image .image_wrapper img")
+        if not image is None:
+            image = image["src"]
+
+        descs = soup.select(".entry-content p")
+        if descs:
+            description = "\n".join([d.text for d in descs])
+        else:
+            description = None
+
+        category = soup.select_one(".event_category").text
+        first_cat = Extractor.remove_accents(category.split(",")[0].lower())
+        tags = []
+        if first_cat in ["grand spectacle"]:
+            category = "Spectacles"
+            tags.append("💃 danse")
+        elif first_cat in ["theatre", "humour / one man show"]:
+            category = "Spectacles"
+            tags.append("🎭 théâtre")
+        elif first_cat in ["chanson francaise", "musique du monde", "pop / rock", "rap", "rnb", "raggae", "variete"]:
+            category = "Fêtes & Concerts"
+            tags.append("🎵 concert")
+        elif first_cat in ["comedie musicale", "humour / one man show", "spectacle equestre"]:
+            category = "Spectacles"
+        elif first_cat in ["spectacle pour enfant"]:
+            tags = ["🎈 jeune public"]
+            category = None
+        else:
+            category = None
+
+        dates = soup.select("#event_ticket_content>ul>li")
+        for d in dates:
+            dt = datetime.datetime.fromisoformat(d.select_one(".date")["content"])
+            date = dt.date()
+            time = dt.time()
+            if str(date) + " " + str(time) in self.possible_dates[event_url]:
+                location = d.select_one(".event_auditory").text
+                
+                self.add_event_with_props(
+                    default_values,
+                    event_url,
+                    title,
+                    category,
+                    date,
+                    location,
+                    description,
+                    tags,
+                    recurrences=None,
+                    uuids=[event_url + "?d=" + str(date) + "&t=" + str(time)],
+                    url_human=event_url,
+                    start_time=time,
+                    end_day=None,
+                    end_time=None,
+                    published=published,
+                    image=image,
+                )

src/agenda_culturel/import_tasks/custom_extractors/c3c.py

@@ -10,11 +10,12 @@ class CExtractor(TwoStepsExtractor):
     def category_c3c2agenda(self, category):
         if not category:
             return None
-        mapping = {"Théâtre": "Théâtre", "Concert": "Concert", "Projection": "Cinéma"}
+        mapping = {"Théâtre": "Spectacles", "Concert": "Fêtes & Concerts", "Projection": "Cinéma"}
+        mapping_tag = {"Théâtre": "🎭 théâtre", "Concert": "🎵 concert", "Projection": None}
         if category in mapping:
-            return mapping[category]
+            return mapping[category], mapping_tag[category]
         else:
-            return None
+            return None, None
 
     def build_event_url_list(self, content):
         soup = BeautifulSoup(content, "html.parser")
@@ -49,20 +50,23 @@ class CExtractor(TwoStepsExtractor):
             
         description = soup.select_one(".presentation").get_text()
         duration = soup.select_one("#criteres .DUREE-V .valeur-critere li")
-        if duration is not None:
+        if not duration is None:
             duration = Extractor.parse_french_time(duration.text)
 
         location = self.nom_lieu
         categories = []
+        tags = []
         for t in soup.select(".sous-titre span"):
             classes = t.get("class")
             if classes and len(classes) > 0:
                 if classes[0].startswith("LIEU-"):
                     location = t.text
                 elif classes[0].startswith("THEMATIQUE-"):
-                    cat = self.category_c3c2agenda(t.text)
-                    if cat is not None:
+                    cat, tag = self.category_c3c2agenda(t.text)
+                    if cat:
                         categories.append(cat)
+                    if tag:
+                        tags.append(tag)
 
         # TODO: parser les dates, récupérer les heures ()
         dates = [o.get("value") for o in soup.select("select.datedleb_resa option")]
@@ -107,21 +111,20 @@ class CExtractor(TwoStepsExtractor):
         self.downloader.pause = pause
 
         category = None
-        if "category" in default_values:
-            category = default_values["category"]
         if len(categories) > 0:
             category = categories[0]
 
         for dt in datetimes:
 
             self.add_event_with_props(
+                default_values,
                 event_url,
                 title,
                 category,
                 dt[0],
                 location,
                 description,
-                [],
+                tags,
                 recurrences=None,
                 uuids=[event_url],
                 url_human=url_human,

src/agenda_culturel/import_tasks/custom_extractors/fbevents.py

@@ -3,6 +3,12 @@ from ..extractor_facebook import FacebookEvent
 import json5
 from bs4 import BeautifulSoup
 import json
+import os
+from datetime import datetime
+
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 # A class dedicated to get events from a facebook events page
@@ -13,10 +19,27 @@ class CExtractor(TwoStepsExtractor):
     def build_event_url_list(self, content):
         soup = BeautifulSoup(content, "html.parser")
 
+        debug = False
+
+        found = False
         links = soup.find_all("a")
         for link in links:
             if link.get("href").startswith('https://www.facebook.com/events/'):
                 self.add_event_url(link.get('href').split('?')[0])
+                found = True
+
+        if not found and debug:
+            directory = "errors/"
+            if not os.path.exists(directory):
+                os.makedirs(directory)
+            now = datetime.now()
+            filename = directory + now.strftime("%Y%m%d_%H%M%S") + ".html"
+            logger.warning("cannot find any event link in events page. Save content page in " + filename)
+            with open(filename, "w") as text_file:
+                text_file.write("<!-- " + self.url + " -->\n\n")
+                text_file.write(content)
+
+                
 
 
     def add_event_from_content(
@@ -41,7 +64,8 @@ class CExtractor(TwoStepsExtractor):
             for event in fevent.build_events(event_url):
                 event["published"] = published
 
-                if "category" in default_values:
-                    event["category"] = default_values["category"]
-                self.add_event(**event)
+                self.add_event(default_values, **event)
+        else:
+            logger.warning("cannot find any event in page")
+
 

src/agenda_culturel/import_tasks/custom_extractors/lacomedie.py

@@ -9,18 +9,31 @@ class CExtractor(TwoStepsExtractor):
     nom_lieu = "La Comédie de Clermont"
     url_referer = "https://lacomediedeclermont.com/saison24-25/"
 
+    def is_to_import_from_url(self, url):
+        if any(keyword in url for keyword in ["podcast", "on-debriefe", "popcorn", "rencontreautour","rencontre-autour"]):
+            return False
+        else:
+            return True
+
     def category_comedie2agenda(self, category):
         mapping = {
-            "Théâtre": "Théâtre",
-            "Danse": "Danse",
-            "Rencontre": "Sans catégorie",
+            "Théâtre": "Spectacles",
+            "Danse": "Spectacles",
+            "Rencontre": "Rencontres & Débats",
             "Sortie de résidence": "Sans catégorie",
             "PopCorn Live": "Sans catégorie",
         }
+        mapping_tag = {
+            "Théâtre": "🎭 théâtre",
+            "Danse": "💃 danse",
+            "Rencontre": None,
+            "Sortie de résidence": "sortie de résidence",
+            "PopCorn Live": None,
+        }
         if category in mapping:
-            return mapping[category]
+            return mapping[category], mapping_tag[category]
         else:
-            return None
+            return None, None
 
     def build_event_url_list(self, content):
         dates = json5.loads(content)["data"][0]
@@ -43,31 +56,35 @@ class CExtractor(TwoStepsExtractor):
                             e_url = (
                                 e.select("a")[0]["href"] + "#" + d
                             )  # a "fake" url specific for each day of this show
-                            self.add_event_url(e_url)
-                            self.add_event_start_day(e_url, d)
-                            t = (
-                                str(e.select("div#datecal")[0])
-                                .split(" ")[-1]
-                                .split("<")[0]
-                            )
-                            self.add_event_start_time(e_url, t)
-                            title = e.select("a")[0].contents[0]
-                            self.add_event_title(e_url, title)
-                            category = e.select("div#lieuevtcal span")
-                            if len(category) > 0:
-                                category = self.category_comedie2agenda(
-                                    category[-1].contents[0]
+
+                            if self.is_to_import_from_url(e_url):
+                                self.add_event_url(e_url)
+                                self.add_event_start_day(e_url, d)
+                                t = (
+                                    str(e.select("div#datecal")[0])
+                                    .split(" ")[-1]
+                                    .split("<")[0]
                                 )
-                                if category is not None:
-                                    self.add_event_category(e_url, category)
-                            location = (
-                                e.select("div#lieuevtcal")[0]
-                                .contents[-1]
-                                .split("•")[-1]
-                            )
-                            if location.replace(" ", "") == "":
-                                location = self.nom_lieu
-                            self.add_event_location(e_url, location)
+                                self.add_event_start_time(e_url, t)
+                                title = e.select("a")[0].contents[0]
+                                self.add_event_title(e_url, title)
+                                category = e.select("div#lieuevtcal span")
+                                if len(category) > 0:
+                                    category, tag = self.category_comedie2agenda(
+                                        category[-1].contents[0]
+                                    )
+                                    if category:
+                                        self.add_event_category(e_url, category)
+                                    if tag:
+                                        self.add_event_tag(e_url, tag)
+                                location = (
+                                    e.select("div#lieuevtcal")[0]
+                                    .contents[-1]
+                                    .split("•")[-1]
+                                )
+                                if location.replace(" ", "") == "":
+                                    location = self.nom_lieu
+                                self.add_event_location(e_url, location)
 
     def add_event_from_content(
         self,
@@ -89,10 +106,22 @@ class CExtractor(TwoStepsExtractor):
         description = soup.select("#descspec")
         if description and len(description) > 0:
             description = description[0].get_text().replace("Lire plus...", "")
+            # on ajoute éventuellement les informations complémentaires
+
+            d_suite = ""
+            for d in ["typedesc", "dureedesc", "lieuspec"]:
+                comp_desc = soup.select("#" + d)
+                if comp_desc and len(comp_desc) > 0:
+                    d_suite += "\n\n" + comp_desc[0].get_text()
+            if d_suite != "":
+                description += "\n\n> Informations complémentaires:" + d_suite
+        else:
+            description = None
 
         url_human = event_url
 
         self.add_event_with_props(
+            default_values,
             event_url,
             None,
             None,

src/agenda_culturel/import_tasks/custom_extractors/lacoope.py

@@ -22,7 +22,7 @@ class CExtractor(TwoStepsExtractor):
             for e in data["events"]:
                 self.add_event_url(e["url"])
                 if e["tag"] == "Gratuit":
-                    self.add_event_tag(e["url"], "gratuit")
+                    self.add_event_tag(e["url"], "💶 gratuit")
 
         else:
             raise Exception("Cannot extract events from javascript")
@@ -38,7 +38,7 @@ class CExtractor(TwoStepsExtractor):
         soup = BeautifulSoup(event_content, "html.parser")
 
         title = soup.find("h1").contents[0]
-        category = "Concert"
+        category = "Fêtes & Concerts"
         image = soup.find("meta", property="og:image")
         if image:
             image = image["content"]
@@ -53,7 +53,7 @@ class CExtractor(TwoStepsExtractor):
         if description is None:
             description = ""
 
-        tags = []
+        tags = ["🎵 concert"]
 
         link_calendar = soup.select('a[href^="https://calendar.google.com/calendar/"]')
         if len(link_calendar) == 0:
@@ -68,6 +68,7 @@ class CExtractor(TwoStepsExtractor):
         url_human = event_url
 
         self.add_event_with_props(
+            default_values,
             event_url,
             title,
             category,

src/agenda_culturel/import_tasks/custom_extractors/lapucealoreille.py

@@ -28,7 +28,7 @@ class CExtractor(TwoStepsExtractor):
 
         title = soup.select("h2")[0].get_text()
 
-        start_day = self.parse_french_date(
+        start_day = Extractor.parse_french_date(
             soup.select("h2")[1].get_text()
         )  # pas parfait, mais bordel que ce site est mal construit
         print(soup.select("h2")[1].get_text())
@@ -58,7 +58,7 @@ class CExtractor(TwoStepsExtractor):
         end_day = Extractor.guess_end_day(start_day, start_time, end_time)
 
         url_human = event_url
-        tags = []
+        tags = ["🎵 concert"]
 
         image = soup.select("wow-image img[fetchpriority=high]")
         if image:
@@ -76,9 +76,10 @@ class CExtractor(TwoStepsExtractor):
             description = None
 
         self.add_event_with_props(
+            default_values,
             event_url,
             title,
-            "Concert",
+            "Fêtes & Concerts",
             start_day,
             location,
             description,

src/agenda_culturel/import_tasks/custom_extractors/lefotomat.py

@@ -9,11 +9,12 @@ class CExtractor(TwoStepsExtractor):
     def category_fotomat2agenda(self, category):
         if not category:
             return None
-        mapping = {"Concerts": "Concert"}
+        mapping = {"Concerts": "Fêtes & Concerts"}
+        mapping_tag = {"Concerts": "🎵 concert"}
         if category in mapping:
-            return mapping[category]
+            return mapping[category], mapping_tag[category]
         else:
-            return None
+            return None, None
 
     def build_event_url_list(self, content):
         soup = BeautifulSoup(content, "xml")
@@ -26,9 +27,11 @@ class CExtractor(TwoStepsExtractor):
             title = e.find("title").contents[0]
             self.add_event_title(e_url, title)
 
-            category = self.category_fotomat2agenda(e.find("category").contents[0])
+            category, tag = self.category_fotomat2agenda(e.find("category").contents[0])
             if category:
                 self.add_event_category(e_url, category)
+            if tag:
+                self.add_event_tag(e_url, tag)
 
     def add_event_from_content(
         self,
@@ -69,6 +72,7 @@ class CExtractor(TwoStepsExtractor):
         url_human = event_url
 
         self.add_event_with_props(
+            default_values,
             event_url,
             None,
             None,

src/agenda_culturel/import_tasks/custom_extractors/lerio.py

@@ -0,0 +1,91 @@
+from ..generic_extractors import *
+from bs4 import BeautifulSoup
+from datetime import datetime
+
+# A class dedicated to get events from Cinéma Le Rio (Clermont-Ferrand)
+# URL: https://www.cinemalerio.com/evenements/
+class CExtractor(TwoStepsExtractorNoPause):
+
+    def __init__(self):
+        super().__init__()
+        self.possible_dates = {}
+        self.theater = None
+
+    def build_event_url_list(self, content, infuture_days=180):
+        
+        soup = BeautifulSoup(content, "html.parser")
+
+        links = soup.select("td.seance_link a")
+        if links:
+            for l in links:
+                print(l["href"])
+                self.add_event_url(l["href"])
+
+    def to_text_select_one(soup, filter):
+        e = soup.select_one(filter)
+        if e is None:
+            return None
+        else:
+            return e.text
+
+    def add_event_from_content(
+        self,
+        event_content,
+        event_url,
+        url_human=None,
+        default_values=None,
+        published=False,
+    ):
+
+        soup = BeautifulSoup(event_content, "html.parser")
+        
+        title = soup.select_one("h1").text
+
+        alerte_date = CExtractor.to_text_select_one(soup, ".alerte_date")
+        if alerte_date is None:
+            return
+        dh = alerte_date.split("à")
+        # if date is not found, we skip
+        if len(dh) != 2:            
+            return
+        
+        date = Extractor.parse_french_date(dh[0], default_year=datetime.now().year)
+        time = Extractor.parse_french_time(dh[1])
+
+        synopsis = CExtractor.to_text_select_one(soup, ".synopsis_bloc")
+        special_titre = CExtractor.to_text_select_one(soup, ".alerte_titre")
+        special = CExtractor.to_text_select_one(soup, ".alerte_text")
+
+        # it's not a specific event: we skip it
+        special_lines = None if special is None else special.split('\n')
+        if special is None or len(special_lines) == 0 or \
+            (len(special_lines) == 1 and special_lines[0].strip().startswith('En partenariat')):
+            return
+
+        description = "\n\n".join([x for x in [synopsis, special_titre, special] if not x is None])
+
+        image = soup.select_one(".col1 img")
+        image_alt = None
+        if not image is None:
+            image_alt = image["alt"]
+            image = image["src"]
+
+        self.add_event_with_props(
+                    default_values,
+                    event_url,
+                    title,
+                    None,
+                    date,
+                    None,
+                    description,
+                    [],
+                    recurrences=None,
+                    uuids=[event_url],
+                    url_human=event_url,
+                    start_time=time,
+                    end_day=None,
+                    end_time=None,
+                    published=published,
+                    image=image,
+                    image_alt=image_alt
+                )

src/agenda_culturel/import_tasks/custom_extractors/wordpress_mec.py

@@ -8,18 +8,28 @@ class CExtractor(TwoStepsExtractor):
     
     def local2agendaCategory(self, category):
         mapping = {
-            "Musique": "Concert",
-            "CONCERT": "Concert",
-            "VISITE": "Sans catégorie",
-            "Spectacle": "Théâtre",
-            "Rencontre": "Sans catégorie",
-            "Atelier": "Sans catégorie",
-            "Projection": "Sans catégorie",
+            "Musique": "Fêtes & Concerts",
+            "CONCERT": "Fêtes & Concerts",
+            "VISITE": "Visites & Expositions",
+            "Spectacle": "Spectacles",
+            "Rencontre": "Rencontres & Débats",
+            "Atelier": "Animations & Ateliers",
+            "Projection": "Cinéma",
         }
+        mapping_tag = {
+            "Musique": "concert",
+            "CONCERT": "concert",
+            "VISITE": None,
+            "Spectacle": "rhéâtre",
+            "Rencontre": None,
+            "Atelier": "atelier",
+            "Projection": None,
+        }
+        
         if category in mapping:
-            return mapping[category]
+            return mapping[category], mapping_tag[category]
         else:
-            return None
+            return None, None
 
     def build_event_url_list(self, content):
         soup = BeautifulSoup(content, "xml")
@@ -39,9 +49,11 @@ class CExtractor(TwoStepsExtractor):
                 if len(categories) == 0:
                     categories = e.select(".mec-category")
                 if len(categories) > 0:
-                    category = self.local2agendaCategory(categories[0].get_text())
-                    if category is not None:
+                    category, tag = self.local2agendaCategory(categories[0].get_text())
+                    if category:
                         self.add_event_category(url, category)
+                    if tag:
+                        self.add_event_category(url, tag)
 
 
     def add_event_from_content(
@@ -81,13 +93,14 @@ class CExtractor(TwoStepsExtractor):
         url_human = event_url
 
         self.add_event_with_props(
+            default_values,
             event_url,
             None,
             None,
             start_day,
-            None if "location" not in default_values else default_values["location"],
-            description,
             None,
+            description,
+            [],
             recurrences=None,
             uuids=[event_url],
             url_human=url_human,

src/agenda_culturel/import_tasks/downloader.py

@@ -53,15 +53,20 @@ class SimpleDownloader(Downloader):
                 resource = urllib.request.urlopen(req, post_args)
             else:
                 resource = urllib.request.urlopen(req)
-            data = resource.read().decode(resource.headers.get_content_charset())
+            charset = resource.headers.get_content_charset()
+            if charset:
+                data = resource.read().decode(charset)
+            else:
+                data = resource.read().decode()
             return data
+
         except Exception as e:
             print(e)
             return None
 
 
 class ChromiumHeadlessDownloader(Downloader):
-    def __init__(self, pause=True):
+    def __init__(self, pause=True, noimage=True):
         super().__init__()
         self.pause = pause
         self.options = Options()
@@ -73,17 +78,31 @@ class ChromiumHeadlessDownloader(Downloader):
         self.options.add_argument("--disable-dev-shm-usage")
         self.options.add_argument("--disable-browser-side-navigation")
         self.options.add_argument("--disable-gpu")
-        self.options.add_experimental_option(
-            "prefs", {
-                # block image loading
-                "profile.managed_default_content_settings.images": 2,
-            }
-        )
+        if noimage:
+            self.options.add_experimental_option(
+                "prefs", {
+                    # block image loading
+                    "profile.managed_default_content_settings.images": 2,
+                }
+            )
 
         self.service = Service("/usr/bin/chromedriver")
         self.driver = webdriver.Chrome(service=self.service, options=self.options)
 
 
+    def screenshot(self, url, path_image):
+        print("Screenshot {}".format(url))
+        try:
+            self.driver.get(url)
+            if self.pause:
+                time.sleep(2)
+            self.driver.save_screenshot(path_image)
+        except:
+            print(f">> Exception: {URL}")
+            return False
+        
+        return True
+
     def download(self, url, referer=None, post=None):
         if post:
             raise Exception("POST method with Chromium headless not yet implemented")

src/agenda_culturel/import_tasks/extractor.py

@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
 from datetime import datetime, time, date, timedelta
 import re
 import unicodedata
-
+from django.utils import timezone
 
 
 
@@ -43,13 +43,13 @@ class Extractor(ABC):
             "nov",
             "dec",
         ]
-        t = remove_accents(text).lower()
+        t = Extractor.remove_accents(text).lower()
         for i, m in enumerate(mths):
             if t.startswith(m):
                 return i + 1
         return None
 
-    def parse_french_date(text):
+    def parse_french_date(text, default_year=None):
         # format NomJour Numero Mois Année
         m = re.search(
             "[a-zA-ZéÉûÛ:.]+[ ]*([0-9]+)[er]*[ ]*([a-zA-ZéÉûÛ:.]+)[ ]*([0-9]+)", text
@@ -73,8 +73,15 @@ class Extractor(ABC):
                     month = int(m.group(2))
                     year = m.group(3)
                 else:
-                    # TODO: consolider les cas non satisfaits
-                    return None
+                    # format Numero Mois Annee
+                    m = re.search("([0-9]+)[er]*[ ]*([a-zA-ZéÉûÛ:.]+)", text)
+                    if m:
+                        day = m.group(1)
+                        month = Extractor.guess_month(m.group(2))
+                        year = default_year
+                    else:
+                        # TODO: consolider les cas non satisfaits
+                        return None
 
         if month is None:
             return None
@@ -152,6 +159,7 @@ class Extractor(ABC):
 
     def add_event(
         self,
+        default_values,
         title,
         category,
         start_day,
@@ -176,14 +184,19 @@ class Extractor(ABC):
             print("ERROR: cannot import an event without start day")
             return
 
+        tags_default = self.default_value_if_exists(default_values, "tags")
+        if not tags_default:
+            tags_default = []
+
         event = {
             "title": title,
-            "category": category,
+            "category": category if category else self.default_value_if_exists(default_values, "category"),
             "start_day": start_day,
             "uuids": uuids,
-            "location": location,
+            "location": location if location else self.default_value_if_exists(default_values, "location"),
+            "organisers": self.default_value_if_exists(default_values, "organisers"),
             "description": description,
-            "tags": tags,
+            "tags": tags + tags_default,
             "published": published,
             "image": image,
             "image_alt": image_alt,
@@ -234,6 +247,28 @@ class Extractor(ABC):
         from .extractor_ggcal_link import GoogleCalendarLinkEventExtractor
 
         if single_event:
-            return [FacebookEventExtractor(), GoogleCalendarLinkEventExtractor()]
+            return [FacebookEventExtractor(), GoogleCalendarLinkEventExtractor(), EventNotFoundExtractor()]
         else:
-            return [ICALExtractor(), FacebookEventExtractor(), GoogleCalendarLinkEventExtractor()]
+            return [ICALExtractor(), FacebookEventExtractor(), GoogleCalendarLinkEventExtractor(), EventNotFoundExtractor()]
+
+
+# A class that only produce a not found event
+class EventNotFoundExtractor(Extractor):
+
+    def extract(
+        self, content, url, url_human=None, default_values=None, published=False
+    ):
+        self.set_header(url)
+        self.clear_events()
+
+        self.add_event(default_values, "événement sans titre depuis " + url, 
+            None, timezone.now().date(), None, 
+            "l'import a échoué, la saisie doit se faire manuellement à partir de l'url source " + url,
+            [],  [url], published=False, url_human=url)
+
+        return self.get_structure()
+
+
+    def clean_url(url):
+        return url
+

src/agenda_culturel/import_tasks/extractor_facebook.py

@@ -237,10 +237,16 @@ class FacebookEventExtractor(Extractor):
         if FacebookEventExtractor.is_known_url(url):
             u = urlparse(url)
             result = "https://www.facebook.com" + u.path
+
+            # remove name in the url
+            match = re.match(r"(.*/events)/s/([a-zA-Z-][a-zA-Z-0-9-]+)/([0-9/]*)", result)
+            if match:
+                result = match[1] + "/" + match[3]
+
             if result[-1] == "/":
-                return result[:-1]
-            else:
                 return result
+            else:
+                return result + "/"
         else:
             return url
 
@@ -272,7 +278,7 @@ class FacebookEventExtractor(Extractor):
 
                 if default_values and "category" in default_values:
                     event["category"] = default_values["category"]
-                self.add_event(**event)
+                self.add_event(default_values, **event)
             return self.get_structure()
         else:
             logger.warning("cannot find any event in page")

src/agenda_culturel/import_tasks/extractor_ggcal_link.py

@@ -19,7 +19,6 @@ class GoogleCalendarLinkEventExtractor(Extractor):
     def extract(
         self, content, url, url_human=None, default_values=None, published=False
     ):
-        # default_values are not used
         soup = BeautifulSoup(content, "html.parser")
 
         for ggu in self.possible_urls:
@@ -41,18 +40,16 @@ class GoogleCalendarLinkEventExtractor(Extractor):
 
                     self.set_header(url)
 
-                    if "category" in default_values:
-                        category = default_values["category"]
-                    else:
-                        category = None
+                    category = None
 
                     self.add_event(
+                        default_values,
                         title=title,
                         category=category,
                         start_day=start_day,
                         location=location,
                         description=description,
-                        tags=None,
+                        tags=[],
                         uuids=[url],
                         recurrences=None,
                         url_human=url_human,

src/agenda_culturel/import_tasks/extractor_ical.py

@@ -78,7 +78,7 @@ class ICALExtractor(Extractor):
 
         for event in calendar.walk("VEVENT"):
             title = self.get_item_from_vevent(event, "SUMMARY")
-            category = self.default_value_if_exists(default_values, "category")
+            category = None
 
             start_day, start_time = self.get_dt_item_from_vevent(event, "DTSTART")
 
@@ -91,8 +91,8 @@ class ICALExtractor(Extractor):
                 end_day = end_day + timedelta(days=-1)
 
             location = self.get_item_from_vevent(event, "LOCATION")
-            if location is None or location.replace(" ", "") == "":
-                location = self.default_value_if_exists(default_values, "location")
+            if (not location is None) and location.replace(" ", "") == "":
+                location = None
 
             description = self.get_item_from_vevent(event, "DESCRIPTION")
             if description is not None:
@@ -127,12 +127,16 @@ class ICALExtractor(Extractor):
                     )
                     # possible limitation: if the ordering is not original then related
 
-            tags = self.default_value_if_exists(default_values, "tags")
+            tags = []
 
             last_modified = self.get_item_from_vevent(event, "LAST-MODIFIED", raw=True)
 
             image = self.guess_image_from_vevent(event)
 
+            url_event = self.get_item_from_vevent(event, "URL", True)
+            if url_event:
+                url_human = url_event
+
             recurrence_entries = {}
             for e in ["RRULE", "EXRULE", "EXDATE", "RDATE"]:
                 i = self.get_item_from_vevent(event, e, raw=True)
@@ -158,6 +162,7 @@ class ICALExtractor(Extractor):
                 if uuidrel is not None:
                     luuids += [uuidrel]
                 self.add_event(
+                    default_values,
                     title,
                     category,
                     start_day,
@@ -182,6 +187,7 @@ class ICALExtractor(Extractor):
 class ICALNoBusyExtractor(ICALExtractor):
     def add_event(
         self,
+        default_values,
         title,
         category,
         start_day,
@@ -199,8 +205,9 @@ class ICALNoBusyExtractor(ICALExtractor):
         image=None,
         image_alt=None,
     ):
-        if title != "Busy" and title != "Accueils bénévoles":
+        if title != "Busy" and title != "Accueils bénévoles" and title != "Occupé":
             super().add_event(
+                default_values,
                 title,
                 category,
                 start_day,
@@ -241,6 +248,7 @@ class ICALNoVCExtractor(ICALExtractor):
 
     def add_event(
         self,
+        default_values,
         title,
         category,
         start_day,
@@ -259,6 +267,7 @@ class ICALNoVCExtractor(ICALExtractor):
         image_alt=None,
     ):
         super().add_event(
+            default_values,
             title,
             category,
             start_day,

src/agenda_culturel/import_tasks/generic_extractors.py

@@ -136,6 +136,7 @@ class TwoStepsExtractor(Extractor):
 
     def add_event_with_props(
         self,
+        default_values,
         event_url,
         title,
         category,
@@ -169,6 +170,7 @@ class TwoStepsExtractor(Extractor):
                 location = self.event_properties[event_url]["location"]
 
         self.add_event(
+            default_values,
             title,
             category,
             start_day,
@@ -212,6 +214,7 @@ class TwoStepsExtractor(Extractor):
         only_future=True,
         ignore_404=True
     ):
+
         self.only_future = only_future
         self.now = datetime.datetime.now().date()
         self.set_header(url)
@@ -239,7 +242,6 @@ class TwoStepsExtractor(Extractor):
                 if ignore_404:
                     logger.error(msg)
                 else:
-                    print("go")
                     raise Exception(msg)
             else:
                 # then extract event information from this html document
@@ -248,3 +250,27 @@ class TwoStepsExtractor(Extractor):
                 )
 
         return self.get_structure()
+
+
+class TwoStepsExtractorNoPause(TwoStepsExtractor):
+
+    def extract(
+        self,
+        content,
+        url,
+        url_human=None,
+        default_values=None,
+        published=False,
+        only_future=True,
+        ignore_404=True
+    ):
+        if hasattr(self.downloader, "pause"):
+            pause = self.downloader.pause
+        else:
+            pause = False
+        self.downloader.pause = False
+        result = super().extract(content, url, url_human, default_values, published, only_future, ignore_404)
+        self.downloader.pause = pause
+
+        return result
+

src/agenda_culturel/import_tasks/importer.py

@@ -1,6 +1,11 @@
 from .downloader import *
 from .extractor import *
 
+import logging
+
+logger = logging.getLogger(__name__)
+
+
 
 class URL2Events:
     def __init__(
@@ -29,8 +34,9 @@ class URL2Events:
         else:
             # if the extractor is not defined, use a list of default extractors
             for e in Extractor.get_default_extractors(self.single_event):
+                logger.warning('Extractor::' + type(e).__name__)
                 e.set_downloader(self.downloader)
                 events = e.extract(content, url, url_human, default_values, published)
-                if events is not None:
+                if events is not None and len(events) > 0:
                     return events
         return None

src/agenda_culturel/migrations/0027_set_dtstart_dtend.py

@@ -20,5 +20,5 @@ class Migration(migrations.Migration):
 
 
     operations = [
-        migrations.RunPython(forwards_func),
+        migrations.RunPython(forwards_func, reverse_code=forwards_func),
     ]

src/agenda_culturel/migrations/0036_auto_20240331_1421.py

@@ -10,6 +10,11 @@ def groups_permissions_creation(apps, schema_editor):
     for name in user_roles:
         Group.objects.create(name=name)
 
+def groups_permissions_deletion(apps, schema_editor):
+    user_roles = ["Automation Manager", "Q&A Manager", "Receptionist"]
+
+    for name in user_roles:
+        Group.objects.filter(name=name).delete()
 
 
 
@@ -21,5 +26,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(groups_permissions_creation),
+        migrations.RunPython(groups_permissions_creation, reverse_code=groups_permissions_deletion),
     ]

src/agenda_culturel/migrations/0038_auto_20240331_1815.py

@@ -31,6 +31,12 @@ def update_groups_permissions(apps, schema_editor):
     Group.objects.get(name="Receptionist").permissions.add(*receptionist_perms)
     Group.objects.get(name="Receptionist").permissions.add(*read_mod_perms)
 
+def update_groups_delete(apps, schema_editor):
+    user_roles = ["Moderator"]
+
+    for name in user_roles:
+        Group.objects.filter(name=name).delete()
+
 
 
 class Migration(migrations.Migration):
@@ -40,5 +46,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(update_groups_permissions),
+        migrations.RunPython(update_groups_permissions, reverse_code=update_groups_delete),
     ]

src/agenda_culturel/migrations/0040_auto_20240403_1924.py

@@ -15,6 +15,9 @@ def update_groups_permissions(apps, schema_editor):
     Group.objects.get(name="Q&A Manager").permissions.add(*qanda_perms)
     Group.objects.get(name="Q&A Manager").permissions.add(*read_mod_perms)
 
+def no_permission_change(apps, schema_editor):
+    pass
+
 
 class Migration(migrations.Migration):
 
@@ -23,5 +26,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(update_groups_permissions),
+        migrations.RunPython(update_groups_permissions, reverse_code=no_permission_change),
     ]

src/agenda_culturel/migrations/0045_auto_20240405_1941.py

@@ -11,7 +11,8 @@ def update_groups_permissions(apps, schema_editor):
     mod_perms = [i for i in all_perms if i.content_type.app_label == 'agenda_culturel' and i.content_type.model == 'moderationquestion' and i.codename.startswith('use_')]
     Group.objects.get(name="Moderator").permissions.add(*mod_perms)
 
-
+def no_permission_change(apps, schema_editor):
+    pass
 
 class Migration(migrations.Migration):
 
@@ -20,5 +21,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(update_groups_permissions),
+        migrations.RunPython(update_groups_permissions, reverse_code=no_permission_change),
     ]

src/agenda_culturel/migrations/0048_auto_20240417_1212.py

@@ -16,6 +16,11 @@ def update_groups_permissions(apps, schema_editor):
     editor_perms = [i for i in all_perms if i.content_type.app_label == 'agenda_culturel' and i.content_type.model == 'staticcontent']
     Group.objects.get(name="Static content editor").permissions.add(*editor_perms)
 
+def update_groups_delete(apps, schema_editor):
+    user_roles = ["Static content editor"]
+
+    for name in user_roles:
+        Group.objects.filter(name=name).delete()
 
 class Migration(migrations.Migration):
 
@@ -24,5 +29,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(update_groups_permissions),
+        migrations.RunPython(update_groups_permissions, reverse_code=update_groups_delete),
     ]

src/agenda_culturel/migrations/0059_auto_20240427_1829.py

@@ -1,9 +1,10 @@
 # Generated by Django 4.2.7 on 2024-04-27 16:29
 
 from django.db import migrations
-from django.contrib.auth.models import Group, Permission
 
 def update_groups_permissions(apps, schema_editor):
+    Group = apps.get_model("auth", "Group")
+    Permission = apps.get_model("auth", "Permission")
 
     all_perms = Permission.objects.all()
 
@@ -11,6 +12,9 @@ def update_groups_permissions(apps, schema_editor):
     moderator_perms = [i for i in all_perms if i.content_type.app_label == 'agenda_culturel' and i.content_type.model in ['place']]
     Group.objects.get(name="Moderator").permissions.add(*moderator_perms)
 
+def no_permission_change(apps, schema_editor):
+    pass
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -18,5 +22,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(update_groups_permissions),
+        migrations.RunPython(update_groups_permissions, reverse_code=no_permission_change),
     ]

src/agenda_culturel/migrations/0081_auto_20241010_2235.py

@@ -1,11 +1,11 @@
 # Generated by Django 4.2.9 on 2024-10-10 20:35
 
 from django.db import migrations
-from agenda_culturel.models import Place
 from django.contrib.gis.geos import Point
 
 def change_coord_format(apps, schema_editor):
-    places = Place.objects.all()
+    Place = apps.get_model("agenda_culturel", "Place")
+    places = Place.objects.values("location", "location_pt").all()
 
     for p in places:
         l = p.location.split(',')
@@ -13,8 +13,15 @@ def change_coord_format(apps, schema_editor):
             p.location_pt = Point(float(l[1]), float(l[0]))
         else:
             p.location_pt = Point(3.08333, 45.783329)
-        p.save()
+        p.save(update_fields=["location_pt"])
 
+def reverse_coord_format(apps, schema_editor):
+    Place = apps.get_model("agenda_culturel", "Place")
+    places = Place.objects.values("location", "location_pt").all()
+
+    for p in places:
+        p.location = ','.join([p.location_pt[1], p.location_pt[0]])
+        p.save(update_fields=["location"])
     
 
 
@@ -25,7 +32,7 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(change_coord_format),
+        migrations.RunPython(change_coord_format, reverse_code=reverse_coord_format),
     ]
 
 

src/agenda_culturel/migrations/0087_referencelocation.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.9 on 2024-10-16 12:55
+
+import django.contrib.gis.geos.point
+from django.db import migrations, models
+import location_field.models.spatial
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0086_alter_recurrentimport_processor'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ReferenceLocation',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(help_text='Name of the location', verbose_name='Name')),
+                ('location', location_field.models.spatial.LocationField(default=django.contrib.gis.geos.point.Point(3.08333, 45.783329), srid=4326)),
+                ('main', models.BooleanField(default=False, help_text='This location is one of the main locations (shown first).', verbose_name='Main')),
+            ],
+        ),
+    ]

src/agenda_culturel/migrations/0088_alter_referencelocation_options_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.9 on 2024-10-16 18:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0087_referencelocation'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='referencelocation',
+            options={'verbose_name': 'Reference location', 'verbose_name_plural': 'Reference locations'},
+        ),
+        migrations.AlterField(
+            model_name='referencelocation',
+            name='name',
+            field=models.CharField(help_text='Name of the location', unique=True, verbose_name='Name'),
+        ),
+    ]

src/agenda_culturel/migrations/0089_alter_recurrentimport_defaultcategory.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-10-17 08:27
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0088_alter_referencelocation_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='recurrentimport',
+            name='defaultCategory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Category of each imported event', null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='agenda_culturel.category', verbose_name='Category'),
+        ),
+    ]

src/agenda_culturel/migrations/0090_alter_recurrentimport_processor.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-10-19 13:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0089_alter_recurrentimport_defaultcategory'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='recurrentimport',
+            name='processor',
+            field=models.CharField(choices=[('ical', 'ical'), ('icalnobusy', 'ical no busy'), ('icalnovc', 'ical no VC'), ('lacoope', 'lacoope.org'), ('lacomedie', 'la comédie'), ('lefotomat', 'le fotomat'), ('lapucealoreille', "la puce à l'oreille"), ('Plugin wordpress MEC', 'Plugin wordpress MEC'), ('Facebook events', "Événements d'une page FB"), ('cour3coquins', 'la cour des 3 coquins'), ('arachnee', 'Arachnée concert')], default='ical', max_length=20, verbose_name='Processor'),
+        ),
+    ]

src/agenda_culturel/migrations/0091_duplicatedevents_fixed_event_masked.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.9 on 2024-10-20 11:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0090_alter_recurrentimport_processor'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='duplicatedevents',
+            name='fixed',
+            field=models.BooleanField(blank=True, default=False, help_text='This duplicated events is fixed, ie exactly one of the listed events is not masked.', null=True, verbose_name='Fixed'),
+        ),
+        migrations.AddField(
+            model_name='event',
+            name='masked',
+            field=models.BooleanField(blank=True, default=False, help_text='This event is masked by a duplicated version.', null=True, verbose_name='Masked'),
+        ),
+    ]

src/agenda_culturel/migrations/0092_alter_categorisationrule_weight.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-10-30 14:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0091_duplicatedevents_fixed_event_masked'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='categorisationrule',
+            name='weight',
+            field=models.IntegerField(default=1, help_text='The lower is the weight, the earlier the filter is applied', verbose_name='Weight'),
+        ),
+    ]

src/agenda_culturel/migrations/0093_tag.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.9 on 2024-10-30 17:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0092_alter_categorisationrule_weight'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Tag',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(help_text='Tag name', max_length=512, verbose_name='Name')),
+                ('description', models.TextField(blank=True, help_text='Description of the tag', null=True, verbose_name='Description')),
+                ('principal', models.BooleanField(default=True, help_text='This tag is highlighted as a main tag for visitors, particularly in the filter.', verbose_name='Principal')),
+            ],
+        ),
+    ]

src/agenda_culturel/migrations/0094_auto_20241030_2002.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.9 on 2024-10-30 19:02
+
+from django.db import migrations
+from django.contrib.auth.models import Group, Permission
+
+def update_groups_permissions(apps, schema_editor):
+    # first add a missing role
+    user_roles = ["Tag editor"]
+
+    for name in user_roles:
+        Group.objects.create(name=name)
+
+    all_perms = Permission.objects.all()
+
+    # set permissions for moderators
+    editor_perms = [i for i in all_perms if i.content_type.app_label == 'agenda_culturel' and i.content_type.model == 'tag']
+    Group.objects.get(name="Tag editor").permissions.add(*editor_perms)
+
+def update_groups_delete(apps, schema_editor):
+    user_roles = ["Tag editor"]
+
+    for name in user_roles:
+        Group.objects.filter(name=name).delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0093_tag'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_groups_permissions, reverse_code=update_groups_delete),
+    ]

src/agenda_culturel/migrations/0095_alter_tag_description.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-10-30 19:11
+
+from django.db import migrations
+import django_ckeditor_5.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0094_auto_20241030_2002'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='description',
+            field=django_ckeditor_5.fields.CKEditor5Field(blank=True, help_text='Description of the tag', null=True, verbose_name='Description'),
+        ),
+    ]

src/agenda_culturel/migrations/0096_alter_tag_name.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-10-30 20:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0095_alter_tag_description'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=models.CharField(help_text='Tag name', max_length=512, unique=True, verbose_name='Name'),
+        ),
+    ]

src/agenda_culturel/migrations/0097_alter_category_alt_name_alter_category_codename.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.9 on 2024-11-01 22:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0096_alter_tag_name'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='category',
+            name='alt_name',
+            field=models.CharField(blank=True, help_text='Alternative name used with a time period', max_length=512, null=True, verbose_name='Alternative Name'),
+        ),
+        migrations.AlterField(
+            model_name='category',
+            name='codename',
+            field=models.CharField(blank=True, help_text='Short name of the category', max_length=3, null=True, verbose_name='Short name'),
+        ),
+    ]

src/agenda_culturel/migrations/0098_remove_category_alt_name_remove_category_codename.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.9 on 2024-11-01 22:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0097_alter_category_alt_name_alter_category_codename'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='category',
+            name='alt_name',
+        ),
+        migrations.RemoveField(
+            model_name='category',
+            name='codename',
+        ),
+    ]

src/agenda_culturel/migrations/0099_update_categories.py

@@ -0,0 +1,202 @@
+# Generated by Django 4.2.9 on 2024-11-01 14:22
+
+from django.db import migrations
+import os.path
+
+class SimpleCat:
+    def __init__(self=None, 
+                 name=None, color=None, 
+                 pictogram=None, position=None, 
+                 transfered_to=None,
+                 transtag=None):
+        self.name = name
+        self.color = color
+        self.pictogram = pictogram
+        self.position = position
+        self.transfered_to = transfered_to
+        self.transfered_to_object = {}
+        self.transtag = transtag
+
+    def get_transfered_category(self, e):
+        # we check if the given event has a corresponding tag (except empty string)
+        if not e is None:
+            for t, c in self.transfered_to.items():
+                if t != "" and t in e.tags:
+                    return c
+        
+        return self.transfered_to[""] if "" in self.transfered_to else None
+
+    def get_transfered_to_object(self, apps, e=None):
+        if self.transfered_to is None:
+            return None, None
+        Category = apps.get_model("agenda_culturel", "Category")
+
+        if isinstance(self.transfered_to, dict):
+            cname = self.get_transfered_category(e)
+        else:
+            cname = self.transfered_to
+
+        if not cname in self.transfered_to_object.keys():
+            self.transfered_to_object[cname] = Category.objects.filter(name=cname).first()
+        
+        return self.transfered_to_object[cname], self.transtag
+    
+    def get_pictogram_file(self):
+        from django.core.files import File
+        f = open(os.path.dirname(__file__) + "/images/" + self.pictogram, "rb")
+        return File(name=self.pictogram, file=f)
+
+# Color selection
+# https://colorkit.co/color-palette-generator/4cae4f-ff9900-2094f3-9b27b0-ffec3d-ff5724-795649-4051b5-009485/
+# #4cae4f, #ff9900, #2094f3, #9b27b0, #ffec3d, #ff5724, #795649, #4051b5, #009485
+
+preserved = {
+    "Nature": { 
+        "old": SimpleCat("Nature", color="#27AEEF", pictogram="leaf.svg", position=8),
+        "new": SimpleCat("Nature", color="#4cae4f", pictogram="leaf.svg", position=8) 
+    },
+    "Cinéma": {
+        "old": SimpleCat("Cinéma", color="#EDE15B", pictogram="theater.svg", position=5),
+        "new": SimpleCat("Cinéma", color="#ff9900", pictogram="theater.svg", position=4),
+    },
+    "Sans catégorie": {
+        "old": SimpleCat("Sans catégorie", color="#AAAAAA", pictogram="calendar.svg", position=100),
+        "new": SimpleCat("Sans catégorie", color="#AAAAAA", pictogram="calendar.svg", position=100),
+    }
+}
+
+
+old_cats = [
+    SimpleCat("Conférence", "#87BC45", "school-outline.svg", 7, "Rencontres & Débats", "conférence"),
+    SimpleCat("Exposition", "#BDCF32", "warehouse.svg", 6, "Visites & Expositions", "exposition"),
+    SimpleCat("Arts du spectacle", "#EDBF33", "track-light.svg", 4, "Spectacles"),
+    SimpleCat("Danse", "#EF9B20", "dance-ballroom.svg", 3, "Spectacles", "danse"),
+    SimpleCat("Concert", "#F46A9B", "account-music-outline.svg", 2, "Fêtes & Concerts", "concert"),
+    SimpleCat("Théâtre", "#EA5545", "drama-masks.svg", 1, "Spectacles", "théâtre")
+]
+
+new_cats = [
+    SimpleCat("Fêtes & Concerts", "#ff5724", "party-popper.svg", 1, {"concert": "Concert", "": "Sans catégorie"}),
+    SimpleCat("Spectacles", "#edbf33", "track-light.svg", 2, {"théâtre": "Théâtre", "danse": "Danse", "": "Arts du spectacle"}),
+    SimpleCat("Rencontres & Débats", "#9b27b0", "workshop.svg", 3, {"conférence": "Conférence", "": "Sans catégorie"}),
+    SimpleCat("Animations & Ateliers", "#4051b5", "tools.svg", 5, "Sans catégorie"),
+    SimpleCat("Rendez-vous locaux", "#2094f3", "ferris-wheel.svg", 6, "Sans catégorie"),
+    SimpleCat("Visites & Expositions", "#795649", "compass-outline.svg", 7, {"exposition": "Exposition", "": "Sans catégorie"}),
+]
+
+def create_categories(apps, catlist):
+    Category = apps.get_model("agenda_culturel", "Category")
+
+    # only create new categories if old ones are present to avoid filling 
+    # an empty database with ghost categories
+    if Category.objects.count() > 1:
+
+        cats = [Category(name=c.name, color=c.color, position=c.position, pictogram=c.get_pictogram_file()) for c in catlist]
+
+        Category.objects.bulk_create(cats)
+
+
+
+def delete_categories(apps, catlist):
+    Category = apps.get_model("agenda_culturel", "Category")
+    Category.objects.filter(name__in=[c.name for c in catlist]).delete()
+
+def create_new_categories(apps, schema_editor):
+    create_categories(apps, new_cats)
+
+def delete_new_categories(apps, schema_editor):
+    delete_categories(apps, new_cats)
+
+def create_old_categories(apps, schema_editor):
+    create_categories(apps, old_cats)
+
+def delete_old_categories(apps, schema_editor):
+    delete_categories(apps, old_cats)
+
+
+
+def update_preserved_categories(apps, dest):
+    other = "old" if dest == "new" else "new"
+    Category = apps.get_model("agenda_culturel", "Category")
+
+    cats = Category.objects.filter(name__in=preserved.keys())
+    ucats = []
+    for c in cats:
+        c.color = preserved[c.name][dest].color
+        c.position = preserved[c.name][dest].position
+        if preserved[c.name][dest].pictogram != preserved[c.name][other].pictogram:
+            c.pictogram = preserved[c.name][dest].get_pictogram_file()
+        ucats.append(c)
+    
+    Category.objects.bulk_update(ucats, fields=["color", "position", "pictogram"])
+
+def update_preserved_categories_new(apps, schema_editor):
+    update_preserved_categories(apps, "new")
+
+def update_preserved_categories_old(apps, schema_editor):
+    update_preserved_categories(apps, "old")
+
+
+def update_database(apps, cats):
+    convert = dict([(c.name, c) for c in cats])
+
+    # update events
+    Event = apps.get_model("agenda_culturel", "Event")
+    events = Event.objects.all()
+    uevents = []
+    for e in events:
+        if e.category and e.category.name in convert.keys():
+            cat, tag = convert[e.category.name].get_transfered_to_object(apps, e)
+            e.category = cat
+            if tag:
+                if e.tags is None:
+                    e.tags = [tag]
+                else:
+                    if not tag in e.tags:
+                        e.tags.append(tag)
+            uevents.append(e)
+    Event.objects.bulk_update(uevents, fields=["category", "tags"])
+
+    # update categorisation rules
+    CategorisationRule = apps.get_model("agenda_culturel", "CategorisationRule")
+    crules = CategorisationRule.objects.all()
+    ucrules = []
+    for r in crules:
+        if r.category and r.category.name in convert.keys():
+            r.category, tag = convert[r.category.name].get_transfered_to_object(apps)
+            ucrules.append(r)
+    CategorisationRule.objects.bulk_update(ucrules, fields=["category"])
+
+    # update recurrent import
+    RecurrentImport = apps.get_model("agenda_culturel", "RecurrentImport")
+    rimports = RecurrentImport.objects.all()
+    urimports = []
+    for ri in rimports:
+        if ri.defaultCategory and ri.defaultCategory.name in convert.keys():
+            ri.defaultCategory, tag = convert[ri.defaultCategory.name].get_transfered_to_object(apps)
+            urimports.append(ri)
+    RecurrentImport.objects.bulk_update(urimports, fields=["defaultCategory"])
+
+
+def update_database_new(apps, schema_editor):
+    update_database(apps, old_cats)
+
+def update_database_old(apps, schema_editor):
+    update_database(apps, new_cats)
+
+def do_nothing(apps, schema_editor):
+    pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0098_remove_category_alt_name_remove_category_codename'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_new_categories, reverse_code=delete_new_categories),
+        migrations.RunPython(update_preserved_categories_new, reverse_code=update_preserved_categories_old),
+        migrations.RunPython(update_database_new, reverse_code=update_database_old),
+        migrations.RunPython(delete_old_categories, reverse_code=create_old_categories)
+    ]
+

src/agenda_culturel/migrations/0100_tag_category.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-11-02 10:54
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0099_update_categories'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='tag',
+            name='category',
+            field=models.ForeignKey(default=None, help_text='This tags corresponds to a sub-category of the given category', null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.category', verbose_name='Category'),
+        ),
+    ]

src/agenda_culturel/migrations/0101_alter_tag_category.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-11-02 14:13
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0100_tag_category'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='category',
+            field=models.ForeignKey(blank=True, default=None, help_text='This tags corresponds to a sub-category of the given category', null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.category', verbose_name='Category'),
+        ),
+    ]

src/agenda_culturel/migrations/0102_duplicatedevents_representative.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-11-07 20:53
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0101_alter_tag_category'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='duplicatedevents',
+            name='representative',
+            field=models.ForeignKey(default=None, help_text='This event is the representative event of the duplicated events group', null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='agenda_culturel.event', verbose_name='Representative event'),
+        ),
+    ]

src/agenda_culturel/migrations/0103_update_duplicatedevents_datastructure.py

@@ -0,0 +1,59 @@
+# Generated by Django 4.2.9 on 2024-11-07 20:53
+
+from django.db import migrations
+
+def set_representative_from_fixed_masked(apps, cats):
+    # get all duplicated events
+    DuplicatedEvents = apps.get_model("agenda_culturel", "DuplicatedEvents")
+    duplicated = DuplicatedEvents.objects.all().prefetch_related('event_set')
+
+    to_update = []
+    for d in duplicated:
+        # there is no representative
+        d.representative = None
+        # except if d is fixed
+        if d.fixed:
+            # and if there is at least one non masked (should be the case)
+            e_not_masked = [e for e in d.event_set.all() if not e.masked]
+            # keep the first one
+            if len(e_not_masked) >= 1:
+                d.representative = e_not_masked[0]
+        to_update.append(d)        
+
+    DuplicatedEvents.objects.bulk_update(to_update, fields=["representative"])
+
+def set_fixed_masked_from_representative(apps, cats):
+    Event = apps.get_model("agenda_culturel", "Event")
+    events = Event.objects.all().prefetch_related("possibly_duplicated")
+
+    to_update = []
+    for e in events:
+        if not e.possibly_duplicated:
+            e.masked = False
+        else:
+            e.masked = e.possibly_duplicated.representative and e.possibly_duplicated.representative == e            
+        to_update.append(e)
+    
+    Event.objects.bulk_update(to_update, fields=["masked"])
+
+    # get all duplicated events
+    DuplicatedEvents = apps.get_model("agenda_culturel", "DuplicatedEvents")
+    duplicated = DuplicatedEvents.objects.all().prefetch_related('event_set')
+
+    # for each event
+    to_update = []
+    for d in duplicated:
+        d.fixed = not d.representative is None
+        to_update.append(d)
+    
+    DuplicatedEvents.objects.bulk_update(to_update, fields=["fixed"])
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0102_duplicatedevents_representative'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_representative_from_fixed_masked, reverse_code=set_fixed_masked_from_representative),
+    ]

src/agenda_culturel/migrations/0104_remove_duplicatedevents_fixed.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.9 on 2024-11-07 21:24
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0103_update_duplicatedevents_datastructure'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='duplicatedevents',
+            name='fixed',
+        ),
+    ]

src/agenda_culturel/migrations/0105_remove_event_masked_remove_event_possibly_duplicated_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.9 on 2024-11-08 08:30
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0104_remove_duplicatedevents_fixed'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='event',
+            name='masked',
+        ),
+        migrations.RenameField(
+            model_name='event',
+            old_name='possibly_duplicated',
+            new_name='other_versions',
+        ),
+    ]

src/agenda_culturel/migrations/0106_alter_event_other_versions.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-11-09 10:43
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0105_remove_event_masked_remove_event_possibly_duplicated_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='event',
+            name='other_versions',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.duplicatedevents', verbose_name='Other versions'),
+        ),
+    ]

src/agenda_culturel/migrations/0107_strip_aliases.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.2.9 on 2024-11-10 21:25
+
+from django.db import migrations
+
+
+def strip_place_aliases(apps, schema_editor):
+    Place = apps.get_model("agenda_culturel", "Place")
+
+    places = Place.objects.all()
+
+    for p in places:
+        if not p.aliases is None:
+            p.aliases = [a.strip() for a in p.aliases]
+
+    Place.objects.bulk_update(places, fields=["aliases"])
+
+    
+
+def do_nothing(apps, schema_editor):
+    pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0106_alter_event_other_versions'),
+    ]
+
+    operations = [
+        migrations.RunPython(strip_place_aliases, reverse_code=do_nothing)
+    ]

src/agenda_culturel/migrations/0108_remove_duplicated_categories.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.2.9 on 2024-11-11 10:15
+
+from django.db import migrations
+
+
+def remove_duplicated_categories(apps, schema_editor):
+    Category = apps.get_model("agenda_culturel", "Category")
+    CategorisationRule = apps.get_model("agenda_culturel", "CategorisationRule")
+    Event = apps.get_model("agenda_culturel", "Event")
+
+
+    catnames = list(set([c.name for c in Category.objects.all()]))
+
+    # for each category name
+    for cname in catnames:
+        # check if it exists more than one category
+        if Category.objects.filter(name=cname).count() > 1:
+            cats = Category.objects.filter(name=cname).order_by("pk")
+            nbs = [Event.objects.filter(category=c).count() + CategorisationRule.objects.filter(category=c).count() for c in cats]
+
+            # if only one category with this name has elements
+            if len([n for n in nbs if n != 0]) == 1:
+                # remove all categories without elements
+                for n, c in zip(nbs, cats):
+                    if n == 0:
+                        c.delete()
+            else:
+                # otherwise, remove all but the last one (by ID)
+                for c in cats[0:-1]:
+                    c.delete()
+
+
+def do_nothing(apps, schema_editor):
+    pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0107_strip_aliases'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_duplicated_categories, reverse_code=do_nothing)
+    ]

src/agenda_culturel/migrations/0109_delete_moderationanswer_delete_moderationquestion.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-11-13 09:56
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0108_remove_duplicated_categories'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='ModerationAnswer',
+        ),
+        migrations.DeleteModel(
+            name='ModerationQuestion',
+        ),
+    ]

src/agenda_culturel/migrations/0110_tag_in_excluded_suggestions_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.9 on 2024-11-13 17:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0109_delete_moderationanswer_delete_moderationquestion'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='tag',
+            name='in_excluded_suggestions',
+            field=models.BooleanField(default=False, help_text='This tag will be part of the excluded suggestions.', verbose_name='In excluded suggestions'),
+        ),
+        migrations.AddField(
+            model_name='tag',
+            name='in_included_suggestions',
+            field=models.BooleanField(default=False, help_text='This tag will be part of the included suggestions.', verbose_name='In included suggestions'),
+        ),
+    ]

src/agenda_culturel/migrations/0111_alter_referencelocation_main.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-11-17 12:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0110_tag_in_excluded_suggestions_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='referencelocation',
+            name='main',
+            field=models.IntegerField(default=0, help_text='This location is one of the main locations (shown first higher values).', verbose_name='Main'),
+        ),
+    ]

src/agenda_culturel/migrations/0112_place_description.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-11-20 15:42
+
+from django.db import migrations
+import django_ckeditor_5.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0111_alter_referencelocation_main'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='place',
+            name='description',
+            field=django_ckeditor_5.fields.CKEditor5Field(blank=True, help_text='Description of the place, including accessibility.', null=True, verbose_name='Description'),
+        ),
+    ]

src/agenda_culturel/migrations/0113_remove_tag_category.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.9 on 2024-11-20 21:40
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0112_place_description'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='tag',
+            name='category',
+        ),
+    ]

src/agenda_culturel/migrations/0114_organisation_event_organisers_and_more.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.2.9 on 2024-11-22 10:12
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_ckeditor_5.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0113_remove_tag_category'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Organisation',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(help_text='Organisation name', max_length=512, unique=True, verbose_name='Name')),
+                ('website', models.URLField(blank=True, help_text='Website of the organisation', max_length=1024, null=True, verbose_name='Website')),
+                ('description', django_ckeditor_5.fields.CKEditor5Field(blank=True, help_text='Description of the organisation.', null=True, verbose_name='Description')),
+                ('principal_place', models.ForeignKey(blank=True, help_text='Place mainly associated with this organizer. Mainly used if there is a similarity in the name, to avoid redundant displays.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.place', verbose_name='Principal place')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='event',
+            name='organisers',
+            field=models.ManyToManyField(blank=True, help_text='list of event organisers. Organizers will only be displayed if one of them does not normally use the venue.', related_name='organised_events', to='agenda_culturel.organisation', verbose_name='Location (free form)'),
+        ),
+        migrations.AddField(
+            model_name='recurrentimport',
+            name='defaultOrganiser',
+            field=models.ForeignKey(blank=True, default=None, help_text='Organiser of each imported event', null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='agenda_culturel.organisation', verbose_name='Organiser'),
+        ),
+    ]

src/agenda_culturel/migrations/0115_alter_organisation_options_alter_event_organisers.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.9 on 2024-11-22 10:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0114_organisation_event_organisers_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='organisation',
+            options={'verbose_name': 'Organisation', 'verbose_name_plural': 'Organisations'},
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='organisers',
+            field=models.ManyToManyField(blank=True, help_text='list of event organisers. Organizers will only be displayed if one of them does not normally use the venue.', related_name='organised_events', to='agenda_culturel.organisation', verbose_name='Organisers'),
+        ),
+    ]

src/agenda_culturel/migrations/0116_event_agenda_cult_start_d_68ab5f_idx.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.9 on 2024-11-23 09:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0115_alter_organisation_options_alter_event_organisers'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='event',
+            index=models.Index(fields=['start_day', 'start_time'], name='agenda_cult_start_d_68ab5f_idx'),
+        ),
+    ]

src/agenda_culturel/migrations/0117_event_start_time_title.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-11-23 10:11
+
+from django.db import migrations, models
+import django.db.models.functions.text
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0116_event_agenda_cult_start_d_68ab5f_idx'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='event',
+            index=models.Index(models.F('start_time'), django.db.models.functions.text.Lower('title'), name='start_time title'),
+        ),
+    ]

src/agenda_culturel/migrations/0118_alter_tag_options_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.9 on 2024-11-23 10:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0117_event_start_time_title'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='tag',
+            options={'verbose_name': 'Étiquette', 'verbose_name_plural': 'Étiquettes'},
+        ),
+        migrations.AddIndex(
+            model_name='category',
+            index=models.Index(fields=['name'], name='agenda_cult_name_28aa03_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='referencelocation',
+            index=models.Index(fields=['name'], name='agenda_cult_name_76f079_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='staticcontent',
+            index=models.Index(fields=['name'], name='agenda_cult_name_fe4995_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='tag',
+            index=models.Index(fields=['name'], name='agenda_cult_name_9c9c74_idx'),
+        ),
+    ]

src/agenda_culturel/migrations/0119_alter_tag_options_alter_event_category_and_more.py

@@ -0,0 +1,74 @@
+# Generated by Django 4.2.9 on 2024-11-27 09:00
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_better_admin_arrayfield.models.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0118_alter_tag_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='tag',
+            options={'verbose_name': 'Tag', 'verbose_name_plural': 'Tags'},
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='category',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='agenda_culturel.category', verbose_name='Category'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='description',
+            field=models.TextField(blank=True, null=True, verbose_name='Description'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='end_day',
+            field=models.DateField(blank=True, null=True, verbose_name='End day'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='end_time',
+            field=models.TimeField(blank=True, null=True, verbose_name='End time'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='exact_location',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='agenda_culturel.place', verbose_name='Location'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='image',
+            field=models.URLField(blank=True, help_text='External URL of the illustration image', max_length=1024, null=True, verbose_name='Illustration (URL)'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='local_image',
+            field=models.ImageField(blank=True, max_length=1024, null=True, upload_to='', verbose_name='Illustration'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='start_day',
+            field=models.DateField(verbose_name='Start day'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='start_time',
+            field=models.TimeField(blank=True, null=True, verbose_name='Start time'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='tags',
+            field=django_better_admin_arrayfield.models.fields.ArrayField(base_field=models.CharField(max_length=64), blank=True, null=True, size=None, verbose_name='Tags'),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='title',
+            field=models.CharField(max_length=512, verbose_name='Title'),
+        ),
+    ]

src/agenda_culturel/migrations/0120_referencelocation_suggested_distance.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-11-27 18:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0119_alter_tag_options_alter_event_category_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='referencelocation',
+            name='suggested_distance',
+            field=models.IntegerField(default=None, help_text='If this distance is given, this location is part of the suggested filters.', null=True, verbose_name='Suggested distance (km)'),
+        ),
+    ]

src/agenda_culturel/migrations/0121_contactmessage_related_event.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-11-27 22:13
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0120_referencelocation_suggested_distance'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='contactmessage',
+            name='related_event',
+            field=models.ForeignKey(default=None, help_text='The message is associated with this event.', null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='agenda_culturel.event', verbose_name='Related event'),
+        ),
+    ]

src/agenda_culturel/migrations/0122_alter_recurrentimport_processor.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-11-29 13:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0121_contactmessage_related_event'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='recurrentimport',
+            name='processor',
+            field=models.CharField(choices=[('ical', 'ical'), ('icalnobusy', 'ical no busy'), ('icalnovc', 'ical no VC'), ('lacoope', 'lacoope.org'), ('lacomedie', 'la comédie'), ('lefotomat', 'le fotomat'), ('lapucealoreille', "la puce à l'oreille"), ('Plugin wordpress MEC', 'Plugin wordpress MEC'), ('Facebook events', "Événements d'une page FB"), ('cour3coquins', 'la cour des 3 coquins'), ('arachnee', 'Arachnée concert'), ('rio', 'Le Rio')], default='ical', max_length=20, verbose_name='Processor'),
+        ),
+    ]

src/agenda_culturel/migrations/0123_event_created_by_user_event_imported_by_user_and_more.py

@@ -0,0 +1,36 @@
+# Generated by Django 4.2.9 on 2024-11-29 18:18
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('agenda_culturel', '0122_alter_recurrentimport_processor'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='event',
+            name='created_by_user',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='created_events', to=settings.AUTH_USER_MODEL, verbose_name='Author of the event creation'),
+        ),
+        migrations.AddField(
+            model_name='event',
+            name='imported_by_user',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='imported_events', to=settings.AUTH_USER_MODEL, verbose_name='Author of the last importation'),
+        ),
+        migrations.AddField(
+            model_name='event',
+            name='moderated_by_user',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='moderated_events', to=settings.AUTH_USER_MODEL, verbose_name='Author of the last moderation'),
+        ),
+        migrations.AddField(
+            model_name='event',
+            name='modified_by_user',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='modified_events', to=settings.AUTH_USER_MODEL, verbose_name='Author of the last modification'),
+        ),
+    ]

src/agenda_culturel/migrations/0124_place_postcode.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-12-06 21:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0123_event_created_by_user_event_imported_by_user_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='place',
+            name='postcode',
+            field=models.CharField(blank=True, help_text='The post code is not displayed, but makes it easier to find an address when you enter it.', null=True, verbose_name='Postcode'),
+        ),
+    ]

src/agenda_culturel/migrations/0125_rename_contactmessage_message_alter_message_options.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.9 on 2024-12-11 11:33
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0124_place_postcode'),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name='ContactMessage',
+            new_name='Message',
+        ),
+        migrations.AlterModelOptions(
+            name='message',
+            options={'verbose_name': 'Message', 'verbose_name_plural': 'Messages'},
+        ),
+    ]

src/agenda_culturel/migrations/0126_message_user.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.9 on 2024-12-11 11:56
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('agenda_culturel', '0125_rename_contactmessage_message_alter_message_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='message',
+            name='user',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to=settings.AUTH_USER_MODEL, verbose_name='Author of the message'),
+        ),
+    ]

src/agenda_culturel/migrations/0127_event_agenda_cult_end_day_4660a5_idx_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.9 on 2024-12-11 19:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0126_message_user'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='event',
+            index=models.Index(fields=['end_day', 'end_time'], name='agenda_cult_end_day_4660a5_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='event',
+            index=models.Index(fields=['status'], name='agenda_cult_status_893243_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='event',
+            index=models.Index(fields=['recurrence_dtstart', 'recurrence_dtend'], name='agenda_cult_recurre_a8911c_idx'),
+        ),
+    ]

src/agenda_culturel/migrations/0128_event_datetimes_title.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-12-11 19:12
+
+from django.db import migrations, models
+import django.db.models.functions.text
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0127_event_agenda_cult_end_day_4660a5_idx_and_more'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='event',
+            index=models.Index(models.F('start_time'), models.F('start_day'), models.F('end_day'), models.F('end_time'), django.db.models.functions.text.Lower('title'), name='datetimes title'),
+        ),
+    ]

src/agenda_culturel/migrations/0129_batchimportation_agenda_cult_created_a23990_idx_and_more.py

@@ -0,0 +1,57 @@
+# Generated by Django 4.2.9 on 2024-12-11 19:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agenda_culturel', '0128_event_datetimes_title'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='batchimportation',
+            index=models.Index(fields=['created_date'], name='agenda_cult_created_a23990_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='batchimportation',
+            index=models.Index(fields=['status'], name='agenda_cult_status_54b205_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='batchimportation',
+            index=models.Index(fields=['created_date', 'recurrentImport'], name='agenda_cult_created_0296e4_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='duplicatedevents',
+            index=models.Index(fields=['representative'], name='agenda_cult_represe_9a4fa2_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='message',
+            index=models.Index(fields=['related_event'], name='agenda_cult_related_79de3c_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='message',
+            index=models.Index(fields=['user'], name='agenda_cult_user_id_42dc88_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='message',
+            index=models.Index(fields=['date'], name='agenda_cult_date_049c71_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='message',
+            index=models.Index(fields=['spam', 'closed'], name='agenda_cult_spam_22f9b3_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='place',
+            index=models.Index(fields=['name'], name='agenda_cult_name_222846_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='place',
+            index=models.Index(fields=['city'], name='agenda_cult_city_156dc7_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='place',
+            index=models.Index(fields=['location'], name='agenda_cult_locatio_6f3c05_idx'),
+        ),
+    ]

src/agenda_culturel/migrations/images/account-music-outline.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m11 4a4 4 0 0 1 4 4 4 4 0 0 1 -4 4 4 4 0 0 1 -4-4 4 4 0 0 1 4-4m0 2a2 2 0 0 0 -2 2 2 2 0 0 0 2 2 2 2 0 0 0 2-2 2 2 0 0 0 -2-2m0 7c1.1 0 2.66.23 4.11.69-.61.38-1.11.91-1.5 1.54-.82-.2-1.72-.33-2.61-.33-2.97 0-6.1 1.46-6.1 2.1v1.1h8.14c.09.7.34 1.34.72 1.9h-10.76v-3c0-2.66 5.33-4 8-4m7.5-3h1.5 2v2h-2v5.5a2.5 2.5 0 0 1 -2.5 2.5 2.5 2.5 0 0 1 -2.5-2.5 2.5 2.5 0 0 1 2.5-2.5c.36 0 .69.07 1 .21z"/></svg>

src/agenda_culturel/migrations/images/calendar.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m19 19h-14v-11h14m-3-7v2h-8v-2h-2v2h-1c-1.11 0-2 .89-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-14c0-1.11-.9-2-2-2h-1v-2m-1 11h-5v5h5z"/></svg>

src/agenda_culturel/migrations/images/compass-outline.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m7 17 3.2-6.8 6.8-3.2-3.2 6.8zm5-5.9a.9.9 0 0 0 -.9.9.9.9 0 0 0 .9.9.9.9 0 0 0 .9-.9.9.9 0 0 0 -.9-.9m0-9.1a10 10 0 0 1 10 10 10 10 0 0 1 -10 10 10 10 0 0 1 -10-10 10 10 0 0 1 10-10m0 2a8 8 0 0 0 -8 8 8 8 0 0 0 8 8 8 8 0 0 0 8-8 8 8 0 0 0 -8-8z"/></svg>

src/agenda_culturel/migrations/images/dance-ballroom.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m14 3.5c0 .83-.67 1.5-1.5 1.5s-1.5-.67-1.5-1.5.67-1.5 1.5-1.5 1.5.67 1.5 1.5m-5.5 1.5c-.83 0-1.5.67-1.5 1.5s.67 1.5 1.5 1.5 1.5-.67 1.5-1.5-.67-1.5-1.5-1.5m5.5 7-.78-2.25h2.96l2.16-1.08c.37-.17.52-.63.33-1-.17-.37-.63-.53-1-.34l-.82.41-.49-.84c-.29-.65-1-1.02-1.7-.86l-2.47.53c-.69.15-1.19.78-1.19 1.5v.7l-2.43 1.62h.01c-.08.07-.19.16-.25.28l-.89 1.77-1.78.89c-.37.17-.52.64-.33 1.01.13.26.4.41.67.41.11 0 .23-.02.34-.08l2.22-1.11 1.04-2.06 1.4 1.5c-1 3-8 7-8 7s4 2 9 2 9-2 9-2-5-4-7-8m2.85-.91-.32.16h-1.2l.06.16c.52 1.03 1.28 2.09 2.11 3.03l-.53-3.41z"/></svg>

src/agenda_culturel/migrations/images/drama-masks.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m8.11 19.45c-2.17-.8-3.89-2.67-4.4-5.1l-1.66-7.81c-.24-1.08.45-2.14 1.53-2.37l9.77-2.07.03-.01c1.07-.21 2.12.48 2.34 1.54l.35 1.67 4.35.93h.03c1.05.24 1.73 1.3 1.51 2.36l-1.66 7.82c-.8 3.77-4.52 6.19-8.3 5.38-1.58-.33-2.92-1.18-3.89-2.34zm11.89-11.27-9.77-2.08-1.66 7.82v.03c-.57 2.68 1.16 5.32 3.85 5.89s5.35-1.15 5.92-3.84zm-4 8.32c-.63 1.07-1.89 1.66-3.17 1.39-1.27-.27-2.18-1.32-2.33-2.55zm-7.53-11.33-4.47.96 1.66 7.81.01.03c.15.71.45 1.35.86 1.9-.1-.77-.08-1.57.09-2.37l.43-2c-.45-.08-.84-.33-1.05-.69.06-.61.56-1.15 1.25-1.31h.25l.78-3.81c.04-.19.1-.36.19-.52m6.56 7.06c.32-.53 1-.81 1.69-.66.69.14 1.19.67 1.28 1.29-.33.52-1 .8-1.7.64-.69-.13-1.19-.66-1.27-1.27m-4.88-1.04c.32-.53.99-.81 1.68-.66.67.14 1.2.68 1.28 1.29-.33.52-1 .81-1.69.68-.69-.17-1.19-.7-1.27-1.31m1.82-6.76 1.96.42-.16-.8z"/></svg>

src/agenda_culturel/migrations/images/ferris-wheel.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m12 19c.86 0 1.59.54 1.87 1.29.55-.12 1.08-.29 1.59-.52l-1.76-4.15c-.52.25-1.09.38-1.7.38s-1.18-.13-1.7-.38l-1.76 4.15c.51.23 1.04.4 1.59.52.28-.75 1.01-1.29 1.87-1.29m6.25-1.24c-.25-.34-.44-.76-.44-1.26 0-1.09.9-2 2-2l.31.03c.25-.8.38-1.65.38-2.53s-.13-1.73-.38-2.5h-.31c-1.11 0-2-.89-2-2 0-.5.16-.91.44-1.26-1.15-1.24-2.66-2.15-4.38-2.53-.28.75-1.01 1.29-1.87 1.29s-1.59-.54-1.87-1.29c-1.72.38-3.23 1.29-4.38 2.53.28.35.45.79.45 1.26 0 1.11-.9 2-2 2h-.32c-.25.78-.38 1.62-.38 2.5 0 .89.14 1.74.39 2.55l.31-.05c1.11 0 2 .92 2 2 0 .5-.16.93-.44 1.27.32.35.68.67 1.05.96l1.9-4.46c-.45-.65-.71-1.43-.71-2.27a4 4 0 0 1 4-4 4 4 0 0 1 4 4c0 .84-.26 1.62-.71 2.27l1.9 4.46c.38-.29.73-.62 1.06-.97m-6.25 5.24c-1 0-1.84-.74-2-1.71-.63-.13-1.25-.34-1.85-.6l-.98 2.31h-2.17l1.41-3.32c-.53-.38-1.02-.82-1.45-1.31-.24.1-.49.13-.76.13a2 2 0 0 1 -2-2c0-.62.3-1.18.77-1.55-.31-.95-.47-1.92-.47-2.95 0-1 .16-2 .46-2.92-.46-.37-.76-.93-.76-1.58 0-1.09.89-2 2-2 .26 0 .51.06.73.15 1.32-1.47 3.07-2.52 5.07-2.94.16-.97 1-1.71 2-1.71s1.84.74 2 1.71c2 .42 3.74 1.47 5.06 2.93.23-.09.48-.14.75-.14a2 2 0 0 1 2 2c0 .64-.31 1.21-.77 1.57.3.93.46 1.93.46 2.93s-.16 2-.46 2.93c.46.37.77.94.77 1.57 0 1.12-.89 2-2 2-.27 0-.52-.04-.76-.14-.44.49-.93.93-1.46 1.32l1.41 3.32h-2.17l-.98-2.31c-.6.26-1.22.47-1.85.6-.16.97-1 1.71-2 1.71z"/></svg>

src/agenda_culturel/migrations/images/leaf.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m17 8c-9 2-11.1 8.17-13.18 13.34l1.89.66.95-2.3c.48.17.98.3 1.34.3 11 0 14-17 14-17-1 2-8 2.25-13 3.25s-7 5.25-7 7.25 1.75 3.75 1.75 3.75c3.25-9.25 13.25-9.25 13.25-9.25z"/></svg>

src/agenda_culturel/migrations/images/party-popper.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m14.53 1.45-1.08 1.08 1.6 1.6c.22.25.33.54.33.87s-.11.64-.33.86l-3.55 3.61 1 1.08 3.63-3.61c.53-.59.79-1.24.79-1.94s-.26-1.36-.79-1.95zm-3.98 2.02-1.08 1.08.61.56c.22.22.33.52.33.89s-.11.67-.33.89l-.61.56 1.08 1.08.56-.61c.53-.59.8-1.23.8-1.92 0-.72-.27-1.37-.8-1.97zm10.45 1.59c-.69 0-1.33.27-1.92.8l-5.63 5.64 1.08 1 5.58-5.56c.25-.25.55-.38.89-.38s.64.13.89.38l.61.61 1.03-1.08-.56-.61c-.59-.53-1.25-.8-1.97-.8m-14 2.94-5 14 14-5zm12 3.06c-.7 0-1.34.27-1.94.8l-1.59 1.59 1.08 1.08 1.59-1.59c.25-.25.53-.38.86-.38s.63.13.88.38l1.62 1.59 1.05-1.03-1.6-1.64c-.59-.53-1.25-.8-1.95-.8z"/></svg>

src/agenda_culturel/migrations/images/school-outline.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m12 3-11 6 4 2.18v6l7 3.82 7-3.82v-6l2-1.09v6.91h2v-8zm6.82 6-6.82 3.72-6.82-3.72 6.82-3.72zm-1.82 7-5 2.72-5-2.72v-3.73l5 2.73 5-2.73z"/></svg>

src/agenda_culturel/migrations/images/theater.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m4 15h2a2 2 0 0 1 2 2v2h1v-2a2 2 0 0 1 2-2h2a2 2 0 0 1 2 2v2h1v-2a2 2 0 0 1 2-2h2a2 2 0 0 1 2 2v2h1v3h-22v-3h1v-2a2 2 0 0 1 2-2m7-8 4 3-4 3zm-7-5h16a2 2 0 0 1 2 2v9.54c-.59-.35-1.27-.54-2-.54v-9h-16v9c-.73 0-1.41.19-2 .54v-9.54a2 2 0 0 1 2-2z"/></svg>

src/agenda_culturel/migrations/images/tools.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m21.71 20.29-1.42 1.42a1 1 0 0 1 -1.41 0l-11.88-11.86a3.81 3.81 0 0 1 -1 .15 4 4 0 0 1 -3.78-5.3l2.54 2.54.53-.53 1.42-1.42.53-.53-2.54-2.54a4 4 0 0 1 5.3 3.78 3.81 3.81 0 0 1 -.15 1l11.86 11.88a1 1 0 0 1 0 1.41m-19.42-1.41a1 1 0 0 0 0 1.41l1.42 1.42a1 1 0 0 0 1.41 0l5.47-5.46-2.83-2.83m12.24-11.42-4 2v2l-2.17 2.17 2 2 2.17-2.17h2l2-4z"/></svg>

src/agenda_culturel/migrations/images/track-light.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m6 1v2h3v3.4l-4.89-2.02-2.68 6.46 5.54 2.3 4.97 3.68 1.85.77 3.83-9.24-1.85-.77-4.77-.71v-3.87h3v-2zm15.81 5.29-2.31.96.76 1.85 2.31-.96zm-2.03 7.28-.78 1.85 2.79 1.15.76-1.85zm-3.59 5.36-1.85.76.96 2.31 1.85-.77z"/></svg>

src/agenda_culturel/migrations/images/warehouse.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m6 19h2v2h-2zm6-16-10 5v13h2v-8h16v8h2v-13zm-4 8h-4v-2h4zm6 0h-4v-2h4zm6 0h-4v-2h4zm-14 4h2v2h-2zm4 0h2v2h-2zm0 4h2v2h-2zm4 0h2v2h-2z"/></svg>

src/agenda_culturel/migrations/images/workshop.svg

@@ -0,0 +1 @@
+<svg height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="m19 7s-5 7-12.5 7c-2 0-5.5 1-5.5 5v4h11v-4c0-2.5 3-1 7-8l-1.5-1.5m-14.5-4.5v-3h20v14h-3m-9-15h4v2h-4zm-4.5 13c1.93299662 0 3.5-1.5670034 3.5-3.5 0-1.93299662-1.56700338-3.5-3.5-3.5s-3.5 1.56700338-3.5 3.5c0 1.9329966 1.56700338 3.5 3.5 3.5z" fill="none" stroke="#000" stroke-width="2"/></svg>

src/agenda_culturel/settings/base.py

@@ -14,7 +14,8 @@ APP_ENV = os_getenv("APP_ENV", "dev")
 DEBUG = os_getenv("DEBUG", "true").lower() in ["True", "true", "1", "yes", "y"]
 
 ALLOWED_HOSTS = os_getenv("ALLOWED_HOSTS", "localhost").split(",")
-
+if DEBUG:
+    ALLOWED_HOSTS = ALLOWED_HOSTS + ['testserver']
 
 if DEBUG:
     CSRF_TRUSTED_ORIGINS = os_getenv("CSRF_TRUSTED_ORIGINS", "http://localhost").split(
@@ -32,6 +33,7 @@ else:
 # Application definition
 
 INSTALLED_APPS = [
+    "agenda_culturel",
     "django.contrib.admin",
     "django.contrib.sitemaps",
     "django.contrib.sites",
@@ -41,7 +43,6 @@ INSTALLED_APPS = [
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "corsheaders",
-    "agenda_culturel",
     "colorfield",
     "django_extensions",
     "django_better_admin_arrayfield",
@@ -55,9 +56,10 @@ INSTALLED_APPS = [
     "robots",
     "debug_toolbar",
     "cache_cleaner",
+    "honeypot",
 ]
 
-SITE_ID = 1
+HONEYPOT_FIELD_NAME = "alias_name"
 
 MIDDLEWARE = [
     "django.middleware.security.SecurityMiddleware",
@@ -71,6 +73,7 @@ MIDDLEWARE = [
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "debug_toolbar.middleware.DebugToolbarMiddleware", 
+    'django.contrib.sites.middleware.CurrentSiteMiddleware',
 #    "django.middleware.cache.UpdateCacheMiddleware",
 #    "django.middleware.common.CommonMiddleware",
 #    "django.middleware.cache.FetchFromCacheMiddleware",
@@ -144,10 +147,9 @@ TIME_ZONE = "Europe/Paris"
 
 USE_I18N = True
 
-USE_TZ = True
+USE_TZ = False
 
 LANGUAGES = (
-    ("en-us", _("English")),
     ("fr", _("French")),
 )
 
@@ -253,4 +255,26 @@ ROBOTS_USE_SITEMAP = False
 if DEBUG:
     import socket
     hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
-    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + ["127.0.0.1", "10.0.2.2"]
+    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + ["127.0.0.1", "10.0.2.2"]
+
+# logging
+
+level_debug = "DEBUG" if DEBUG else "ERROR"
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {
+        "file": {
+            "level": level_debug,
+            "class": "logging.FileHandler",
+            "filename": "backend.log",
+        },
+    },
+    "loggers": {
+        "django": {
+            "handlers": ["file"],
+            "level": level_debug,
+            "propagate": True,
+        },
+    },
+}

src/agenda_culturel/sitemaps.py

@@ -0,0 +1,13 @@
+from django.contrib import sitemaps
+from django.urls import reverse
+
+
+class StaticViewSitemap(sitemaps.Sitemap):
+    priority = 0.5
+    changefreq = "daily"
+
+    def items(self):
+        return ["home", "cette_semaine", "ce_mois_ci", "aujourdhui", "a_venir", "about", "contact"]
+
+    def location(self, item):
+        return reverse(item)