daemon: Remove birdnet_miner and call birdnet_output_to_sql at each new model execution

CHANGELOG.md

@@ -3,8 +3,9 @@
 
 - Add docker compose port
 - Improve install script
+- Add base uninstall script (need deeper work)
 - Add ttyd for systemd logging
-## v0.0.1-rc
+## v0.0.1-rc (2022-08-18)
 
 - Integrate BirdNET-Analyzer as submodule
 - Add birdnet_recording service
@@ -15,4 +16,4 @@
 - Add /today/species and /today/{date}/species/{id} endpoints
 - Add records deletion button and /records/delete endpoint as well as bulk deletion (select all button on /today/species/{id} endpoint)
 - Add systemd status page /status
-- Add i18n for webapp (not species name), en|fr only for the moment
+- Add i18n for webapp (not species name), en|fr only for the moment

INSTALL.md

@@ -129,7 +129,7 @@ sudo mv /composer.phar /usr/local/bin/composer
 
 ```bash
 cd www
-composer install
+composer install --no-dev --prefer-dist --optimize-autoloader
 ```
 
 ### Install nodejs and npm
@@ -147,7 +147,7 @@ nvm use 16
 ```
 
 ```bash
-sudo dnf install npm
+sudo apt-get install npm
 ```
 
 ```bash

daemon/birdnet_analyzis.sh

@@ -3,9 +3,7 @@ set -e
 
 DEBUG=${DEBUG:-1}
 debug() {
-    if [ $DEBUG -eq 1 ]; then
+    [[ $DEBUG -eq 1 ]] && echo "$@"
-        echo "$1"
-    fi
 }
 
 config_filepath="./config/birdnet.conf"
@@ -64,7 +62,9 @@ check_prerequisites() {
 
 # Get array of audio chunks to be processed
 get_chunk_list() {
-    find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0 | sort
+    chunk_list=($(ls ${CHUNK_FOLDER}/in))
+    echo "${chunk_list}"
+    # find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0 | sort
 }
 
 # Perform audio chunk analysis on one chunk
@@ -75,13 +75,22 @@ analyze_chunk() {
     mkdir -p "$output_dir"
     date=$(echo $chunk_name | cut -d'_' -f2)
     week=$(./daemon/weekof.sh $date)
-    $PYTHON_EXECUTABLE ./analyzer/analyze.py --i $chunk_path --o "$output_dir/model.out.csv" --lat $LATITUDE --lon $LONGITUDE --week $week --min_conf $CONFIDENCE --threads 4 --rtype csv
+    if [[ ! -z "${THREADS}" ]]; then
+        threads="--threads ${THREADS}"
+    else 
+        threads=""
+    fi
+    $PYTHON_EXECUTABLE ./analyzer/analyze.py --i $chunk_path --o "$output_dir/model.out.csv" --lat $LATITUDE --lon $LONGITUDE --week $week --min_conf $CONFIDENCE $threads --rtype csv
     debug "Model output written to $output_dir/model.out.csv"
+    bash ./daemon/birdnet_output_to_sql.sh "$output_dir/model.out.csv"
+    debug "Dumped to SQL database"
 }
 
 # Perform audio chunk analysis on all recorded chunks
 analyze_chunks() {
-    for chunk_name in $(get_chunk_list); do
+    local chunks
+    chunks="${1}"
+    for chunk_name in "${chunks}"; do
         if [[ -f "${CHUNK_FOLDER}/out/$chunk_name.d/model.out.csv" ]]; then
             debug "Skipping $chunk_name, as it has already been analyzed"
         else
@@ -98,4 +107,4 @@ check_prerequisites
 chunks=$(get_chunk_list)
 
 # Analyze all chunks in working directory
-analyze_chunks $chunks
+analyze_chunks "$chunks"

daemon/birdnet_output_to_sql.sh

@@ -1,15 +1,13 @@
 #! /usr/bin/env bash
-# Extract observations from a model output folder
+# Extract observations from a model output file into SQL database
 #
 
 DEBUG=${DEBUG:-1}
 set -e
 # set -x
-
+DEBUG=${DEBUG:-1}
 debug() {
-    if [ $DEBUG -eq 1 ]; then
+    [[ $DEBUG -eq 1 ]] && echo "$@"
-        echo "$1"
-    fi
 }
 
 # Load bash library to deal with BirdNET-stream database
@@ -18,16 +16,6 @@ source ./daemon/database/scripts/database.sh
 # Load config
 source ./config/birdnet.conf
 # Check config
-if [[ -z ${CHUNK_FOLDER} ]]; then
-    echo "CHUNK_FOLDER is not set"
-    exit 1
-else
-    if [[ ! -d ${CHUNK_FOLDER}/out ]]; then
-        echo "CHUNK_FOLDER does not exist: ${CHUNK_FOLDER}/out"
-        echo "Cannot extract observations."
-        exit 1
-    fi
-fi
 
 if [[ -z ${LATITUDE} ]]; then
     echo "LATITUDE is not set"
@@ -39,10 +27,6 @@ if [[ -z ${LONGITUDE} ]]; then
     exit 1
 fi
 
-model_outputs() {
-    ls ${CHUNK_FOLDER}/out/*/model.out.csv
-}
-
 source_wav() {
     model_output_path=$1
     model_output_dir=$(dirname $model_output_path)
@@ -107,13 +91,6 @@ save_observations() {
     done
 }
 
-main() {
+model_output_path="$1"
-    # # Remove all junk observations
-    # ./daemon/birdnet_clean.sh
-    # Get model outputs
-    for model_output in $(model_outputs); do
-        save_observations $model_output
-    done
-}
 
-main
+save_observations $model_output_path

daemon/plotter/chart.py

@@ -6,90 +6,119 @@ import matplotlib.pyplot as plt
 from matplotlib.colors import LogNorm
 import seaborn as sns
 from datetime import datetime
+import os
+import glob
 
 CONFIG = {
     "readings": 10,
     "palette": "Greens",
     "db": "./var/db.sqlite",
-    "date": datetime.now().strftime("%Y-%m-%d")
+    "date": datetime.now().strftime("%Y-%m-%d"),
-    # "date": "2022-08-15"
+    "charts_dir": "./var/charts"
 }
 
-db = sqlite3.connect(CONFIG['db'])
+db = None
 
-df = pd.read_sql_query("""SELECT common_name, date, location_id, confidence 
+def get_database():
-                        FROM observation
+    global db
-                        INNER JOIN taxon 
+    if db is None:
-                        ON observation.taxon_id = taxon.taxon_id""", db)
+        db = sqlite3.connect(CONFIG["db"])
-df['date'] = pd.to_datetime(df['date'])
+    return db
-df['hour'] = df['date'].dt.hour
-df['date'] = df['date'].dt.date
-df['date'] = df['date'].astype(str)
-df_on_date = df[df['date'] == CONFIG['date']]
- 
-top_on_date = (df_on_date['common_name'].value_counts()[:CONFIG['readings']])
-if top_on_date.empty:
-    print("No observations on {}".format(CONFIG['date']))
-    exit()
     
-df_top_on_date = df_on_date[df_on_date['common_name'].isin(top_on_date.index)]
+def chart(date):
+    db  = get_database()
+    df = pd.read_sql_query(f"""SELECT common_name, date, location_id, confidence 
+                            FROM observation
+                            INNER JOIN taxon 
+                            ON observation.taxon_id = taxon.taxon_id 
+                            WHERE STRFTIME("%Y-%m-%d", `date`) = '{date}'""", db)
+    df['date'] = pd.to_datetime(df['date'])
+    df['hour'] = df['date'].dt.hour
+    df['date'] = df['date'].dt.date
+    df['date'] = df['date'].astype(str)
+    df_on_date = df[df['date'] == date]
+    
+    top_on_date = (df_on_date['common_name'].value_counts()[:CONFIG['readings']])
+    if top_on_date.empty:
+        print("No observations on {}".format(date))
+        return
+    else:
+        print(f"Found observations on {date}")
+        
+    df_top_on_date = df_on_date[df_on_date['common_name'].isin(top_on_date.index)]
 
-# Create a figure with 2 subplots
+    # Create a figure with 2 subplots
-fig, axs = plt.subplots(1, 2, figsize=(20, 5), gridspec_kw=dict(
+    fig, axs = plt.subplots(1, 2, figsize=(20, 5), gridspec_kw=dict(
-    width_ratios=[2, 6]))
+        width_ratios=[2, 6]))
-plt.subplots_adjust(left=None, bottom=None, right=None,
+    plt.subplots_adjust(left=None, bottom=None, right=None,
-                    top=None, wspace=0, hspace=0)
+                        top=None, wspace=0, hspace=0)
 
-# Get species frequencies
+    # Get species frequencies
-frequencies_order = pd.value_counts(df_top_on_date['common_name']).iloc[:CONFIG['readings']].index
+    frequencies_order = pd.value_counts(df_top_on_date['common_name']).iloc[:CONFIG['readings']].index
-# Get min max confidences
+    # Get min max confidences
-confidence_minmax = df_top_on_date.groupby('common_name')['confidence'].max()
+    confidence_minmax = df_top_on_date.groupby('common_name')['confidence'].max()
-confidence_minmax = confidence_minmax.reindex(frequencies_order)
+    confidence_minmax = confidence_minmax.reindex(frequencies_order)
-# Norm values for color palette
+    # Norm values for color palette
-norm = plt.Normalize(confidence_minmax.values.min(),
+    norm = plt.Normalize(confidence_minmax.values.min(),
-                        confidence_minmax.values.max())
+                            confidence_minmax.values.max())
 
-colors = plt.cm.Greens(norm(confidence_minmax))
+    colors = plt.cm.Greens(norm(confidence_minmax))
-plot = sns.countplot(y='common_name', data=df_top_on_date, palette=colors, order=frequencies_order, ax=axs[0])
+    plot = sns.countplot(y='common_name', data=df_top_on_date, palette=colors, order=frequencies_order, ax=axs[0])
 
-plot.set(ylabel=None)
+    plot.set(ylabel=None)
-plot.set(xlabel="Detections")
+    plot.set(xlabel="Detections")
 
-heat = pd.crosstab(df_top_on_date['common_name'], df_top_on_date['hour'])
+    heat = pd.crosstab(df_top_on_date['common_name'], df_top_on_date['hour'])
-# Order heatmap Birds by frequency of occurrance
+    # Order heatmap Birds by frequency of occurrance
-heat.index = pd.CategoricalIndex(heat.index, categories=frequencies_order)
+    heat.index = pd.CategoricalIndex(heat.index, categories=frequencies_order)
-heat.sort_index(level=0, inplace=True)
+    heat.sort_index(level=0, inplace=True)
 
-hours_in_day = pd.Series(data=range(0, 24))
+    hours_in_day = pd.Series(data=range(0, 24))
-heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
+    heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
-heat = (heat + heat_frame).fillna(0)
+    heat = (heat + heat_frame).fillna(0)
 
-# Generate heatmap plot
+    # Generate heatmap plot
-plot = sns.heatmap(
+    plot = sns.heatmap(
-    heat,
+        heat,
-    norm=LogNorm(),
+        norm=LogNorm(),
-    annot=True,
+        annot=True,
-    annot_kws={
+        annot_kws={
-        "fontsize": 7
+            "fontsize": 7
-    },
+        },
-    fmt="g",
+        fmt="g",
-    cmap=CONFIG['palette'],
+        cmap=CONFIG['palette'],
-    square=False,
+        square=False,
-    cbar=False,
+        cbar=False,
-    linewidth=0.5,
+        linewidth=0.5,
-    linecolor="Grey",
+        linecolor="Grey",
-    ax=axs[1],
+        ax=axs[1],
-    yticklabels=False)
+        yticklabels=False)
-plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
+    plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
 
-for _, spine in plot.spines.items():
+    for _, spine in plot.spines.items():
-    spine.set_visible(True)
+        spine.set_visible(True)
 
-plot.set(ylabel=None)
+    plot.set(ylabel=None)
-plot.set(xlabel="Hour of day")
+    plot.set(xlabel="Hour of day")
-plt.suptitle(f"Top {CONFIG['readings']} species on {CONFIG['date']}", fontsize=14)
+    plt.suptitle(f"Top {CONFIG['readings']} species on {CONFIG['date']}", fontsize=14)
-plt.text(15, 11, f"(Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})")
+    plt.text(15, 11, f"(Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})")
-plt.savefig(f"./var/charts/chart_{CONFIG['date']}.png", dpi=300)
+    plt.savefig(f"./var/charts/chart_{CONFIG['date']}.png", dpi=300)
-plt.close()
+    print(f"Plot for {date} saved.")
+    plt.close()
 
-db.close()
+def main():
+    done_charts = glob.glob(f"{CONFIG['charts_dir']}/*.png")
+    last_modified = max(done_charts, key=os.path.getctime)
+    last_modified_date = last_modified.split("_")[-1].split(".")[0]
+    missing_dates = pd.date_range(start=last_modified_date, end=CONFIG['date'], freq='D')
+    print(missing_dates)
+    for missing_date in missing_dates:
+        date = missing_date.strftime("%Y-%m-%d")
+        chart(date)
+    chart(CONFIG['date'])
+    if db is not None:
+        db.close()
+    print("Done.")
+
+if __name__ == "__main__":
+    main()

daemon/systemd/templates/birdnet_miner.service

@@ -1,13 +0,0 @@
-[Unit]
-Description=BirdNET-stream miner service
-
-[Service]
-Type=simple
-User=<USER>
-Group=<GROUP>
-WorkingDirectory=<DIR>
-ExecStart=bash ./daemon/birdnet_miner.sh
-RemainAfterExit=yes
-
-[Install]
-WantedBy=multi-user.target

daemon/systemd/templates/birdnet_miner.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=BirdNET-stream miner Timer
-
-[Timer]
-OnCalendar=*:0/15
-Unit=birdnet_miner.service
-
-[Install]
-WantedBy=timers.target

docs/DOCKER.md

@@ -34,10 +34,13 @@ Then, create your dotenv file and populate it with your own configuration (for i
 cp .env.example .env
 ```
 
-Then, run docker-compose:
+You may need to adapt the listening ports of the services or other configuration parameters. 
+In general all variables stated with ${VARIABLE:-default} inside [../docker-compose.yml](../docker-compose.yml) can be override in the .env file using `VARIABLE=value`.
+
+Once that is done, you can build and start docker services:
 
 ```bash
-# Build image (first time only)
+# Build images (first time only)
 docker compose build
 # Run
 docker compose up # add `-d`, to run in background
@@ -48,4 +51,4 @@ docker compose down
 For a one liner:
 ```bash
 docker compose up --build
-```
+```

install.sh

@@ -67,7 +67,7 @@ install_birdnetstream_services() {
     DIR="$WORKDIR"
     cd "$WORKDIR"
     debug "Setting up BirdNET stream systemd services"
-    services="birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_miner.service birdnet_plotter.service birdnet_plotter.timer"
+    services="birdnet_recording.service birdnet_analyzis.service birdnet_plotter.service birdnet_plotter.timer"
     read -r -a services_array <<<"$services"
     for service in ${services_array[@]}; do
         sudo cp "daemon/systemd/templates/$service" "/etc/systemd/system/"
@@ -78,7 +78,7 @@ install_birdnetstream_services() {
     done
     sudo sed -i "s|<VENV>|$WORKDIR/$PYTHON_VENV|g" "/etc/systemd/system/birdnet_plotter.service"
     sudo systemctl daemon-reload
-    enabled_services="birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_plotter.timer"
+    enabled_services="birdnet_recording.service birdnet_analyzis.service birdnet_plotter.timer"
     read -r -a services_array <<<"$services"
     for service in ${services_array[@]}; do
         debug "Enabling $service"
@@ -163,7 +163,7 @@ setup_http_server() {
     fi
     debug "Enable birdnet.lan domain"
     sudo ln -s /etc/nginx/sites-available/birdnet-stream.conf /etc/nginx/sites-enabled/birdnet-stream.conf
-    debug "Info: Please edit /etc/nginx/sites-available/birdnet-stream.conf to set the correct server name and paths"
+    debug "INFO: Please edit /etc/nginx/sites-available/birdnet-stream.conf to set the correct server name and paths"
     debug "Setup nginx variables the best way possible"
     sudo sed -i "s|<SYMFONY_PUBLIC>|$WORKDIR/www/public/|g" /etc/nginx/sites-available/birdnet-stream.conf
     sudo sed -i "s|<RECORDS_DIR>|$CHUNK_FOLDER/out|g" /etc/nginx/sites-available/birdnet-stream.conf

uninstall.sh

@@ -38,3 +38,12 @@ uninstall_webapp() {
     sudo unlink /etc/nginx/sites-enabled/birdnet-stream.conf
     sudo systemctl restart nginx
 }
+
+main() {
+    echo "WARNING: This will remove all BirdNET-stream related files and services. \
+    Note that it may forget some special configuration."
+    uninstall_webapp
+    uninstall_birdnet_services
+}
+
+main

www/src/Controller/HomeController.php

@@ -25,11 +25,15 @@ class HomeController extends AbstractController
      * @Route("", name="home")
      * @Route("/{_locale<%app.supported_locales%>}/", name="home_i18n")
      */
-    public function index()
+    public function index(Request $request)
     {
+        $date = $request->get("on");
+        if ($date == null) {
+            $date = date("Y-m-d");
+        }
         return $this->render('index.html.twig', [
-            "stats" => $this->get_stats(),
+            "stats" => $this->get_stats($date),
-            "charts" => $this->last_chart_generated(),
+            "charts" => $this->last_chart_generated($date),
         ]);
     }
 
@@ -42,11 +46,12 @@ class HomeController extends AbstractController
         return $this->render('about/index.html.twig', []);
     }
 
-    private function get_stats()
+    private function get_stats($date)
     {
         $stats = array();
         $stats["most-recorded-species"] = $this->get_most_recorded_species();
         $stats["last-detected-species"] = $this->get_last_recorded_species();
+        $stats["number-of-species-detected"] = $this->get_number_of_species_detected($date);
         return $stats;
     }
 
@@ -86,6 +91,27 @@ class HomeController extends AbstractController
         return $species;
     }
 
+    private function get_number_of_species_detected($date)
+    {
+        $count = 0;
+        $sql = "SELECT COUNT(`taxon_id`) AS contact_count
+                FROM `observation`
+                WHERE STRFTIME('%Y-%m-%d', `date`) = :date
+                GROUP BY `taxon_id`";
+        try {
+            $stmt = $this->connection->prepare($sql);
+            $stmt->bindValue(":date", $date);
+            $result = $stmt->executeQuery();
+            $output = $result->fetchAllAssociative();
+            if ($output != null) {
+                $count = $output[0]["contact_count"];
+            }
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
+        return $count;
+    }
+
     private function last_chart_generated()
     {
         $files = glob($this->getParameter('kernel.project_dir') . '/../var/charts/*.png');

www/templates/stats.html.twig

@@ -1,50 +1,78 @@
 <div id="stats">
-    <h2>{{ "Quick Stats" | trans }}</h2>
+  <h2>
-    <ul>
+    {{ 'Quick Stats'|trans }}
-        <li class="most-recorded-species">
+  </h2>
-            {{ "Most recorded species" | trans }}:
+  <ul>
-            {% if stats["most-recorded-species"] is defined and stats["most-recorded-species"]|length > 0 %}
+    <li class="stat">
-            <span class="scientific-name">
+      {{ 'Most recorded species'|trans }}:{% if
-                {{ stats["most-recorded-species"]["scientific_name"] }}
+        stats['most-recorded-species'] is defined
-            </span>
+          and (stats['most-recorded-species']|length) > 0 %}
-            (<span class="common_name">{{ stats["most-recorded-species"]["common_name"] }}</span>)
+        <span class="scientific-name">
-            {{ "with" | trans }}
+          {{ stats['most-recorded-species']['scientific_name'] }}
-            <span class="observation-count">
+        </span>
-                {{ stats["most-recorded-species"]["contact_count"] }}
+        (<span class="common_name">
-            </span>
+          {{ stats['most-recorded-species']['common_name'] }}
-            {{ "contacts" | trans }}.
+        </span>)
-            {% else %}
+        {{ 'with'|trans }}
-                {{ "No species in database." | trans }}
+        <span class="observation-count">
-            {% endif %}
+          {{ stats['most-recorded-species']['contact_count'] }}
-        </li>
+        </span>
-        <li class="last-recorded-species">
+        {{ 'contacts'|trans }}.
-            {{ "Last detected species" | trans }}:
+      {% else %}
-            {% if stats["last-detected-species"] is defined and stats["last-detected-species"]|length > 0 %}
+        {{ 'No species in database.'|trans }}
-            <span class="scientific-name">
+      {% endif %}
-                {{ stats["last-detected-species"]["scientific_name"] }}
+    </li>
-            </span>
+    <li class="stat">
-            (<span class="common_name">{{ stats["last-detected-species"]["common_name"] }}</span>)
+      {{ 'Last detected species'|trans }}:{% if
-            {{ "with" | trans }}
+        stats['last-detected-species'] is defined
-            <span class="confidence">
+          and (stats['last-detected-species']|length) > 0 %}
-                {{ stats["last-detected-species"]["confidence"] }}
+        <span class="scientific-name">
-            </span>
+          {{ stats['last-detected-species']['scientific_name'] }}
-            {{ "AI confidence" | trans }}
+        </span>
-            <span class="datetime">
+        (<span class="common_name">
-            {% set date = stats["last-detected-species"]["date"] %}
+          {{ stats['last-detected-species']['common_name'] }}
-            {% if  date | date("Y-m-d") == "now" | date("Y-m-d") %}
+        </span>)
-                {{ "today" | trans }}
+        {{ 'with'|trans }}
-            {% else %}
+        <span class="confidence">
-                {{ "on" | trans }}
+          {{ stats['last-detected-species']['confidence'] }}
-                {{ date | format_datetime("full", "none") }}
+        </span>
-            {% endif %}
+        {{ 'AI confidence'|trans }}
-            at 
+        <span class="datetime">
-            <span class="time">
+          {% set date = stats['last-detected-species']['date'] %}
-                {{ date | date("H:i") }} 
+          {% if (date|date('Y-m-d')) == ('now'|date('Y-m-d')) %}
-            </span>
+            {{ 'today'|trans }}
-            </span>.
+          {% else %}
-            {% else %}
+            {{ 'on'|trans }}
-                {{ "No species in database" | trans }}
+            {{ date|format_datetime('full', 'none') }}
-            {% endif %}
+          {% endif %}at
-        </li>
+          <span class="time">{{ date|date('H:i') }}</span>
-    </ul>
+        </span>.
-</div>
+      {% else %}
+        {{ 'No species in database'|trans }}
+      {% endif %}
+    </li>
+    <li class="stat">
+      {% set today = 'now'|date('Y-m-d') %}
+      {% set date = app.request.get('on') %}
+      {% if
+        stats['number-of-species-detected'] is defined
+          and stats['number-of-species-detected'] > 0 %}
+        {% if today == date %}
+          {{ 'Number of species detected today: '|trans }}
+        {% else %}
+          {{ 'Number of species detected on '|trans }}
+          {{ date|format_datetime('full', 'none') }}:
+        {% endif %}
+        <span>{{ stats['number-of-species-detected'] }}</span>.
+      {% else %}
+        {# {{ 'No species detected today'|trans }} #}
+        {% if today == date %}
+          {{ 'No species detected today.'|trans }}
+        {% else %}
+          {{ 'No species detected on '|trans }}
+          {{ date|format_datetime('full', 'none') }}
+        {% endif %}
+      {% endif %}
+    </li>
+  </ul>
+</div>