From acc51fdfc4b00977f077b5fbf48d3d8fb2637e96 Mon Sep 17 00:00:00 2001 From: Samuel ORTION Date: Tue, 16 Aug 2022 05:21:53 +0200 Subject: [PATCH] Add plotter and fix miner --- .gitignore | 2 +- .ideas/birdnet_obs_miner.sh | 35 ---- .ideas/journal.php | 2 + .ideas/url_escape.php | 3 + TODO | 3 + config/analyzer.conf | 8 + config/species_list.txt | 189 ------------------ daemon/birdnet_analyzis.sh | 2 + daemon/birdnet_manager.sh | 26 ++- daemon/birdnet_miner.sh | 4 +- daemon/birdnet_recording.sh | 2 +- daemon/notify/apprise.sh | 13 ++ daemon/plotter/__init__.py | 0 daemon/plotter/chart.py | 120 +++++++++++ .../templates/birdnet_analyzis.service | 4 +- .../systemd/templates/birdnet_miner.service | 4 +- daemon/systemd/templates/birdnet_miner.timer | 2 +- .../systemd/templates/birdnet_plotter.service | 12 ++ .../systemd/templates/birdnet_plotter.timer | 9 + .../templates/birdnet_recording.service | 2 +- install.sh | 4 +- requirements.txt | 11 + www/assets/styles/app.css | 19 +- www/assets/styles/menu.css | 1 + www/src/Controller/LogsController.php | 38 ++++ www/src/Controller/ServicesController.php | 60 ++++-- www/templates/index.html.twig | 2 +- www/templates/logs/logs.html.twig | 14 ++ www/templates/menu.html.twig | 28 +-- www/templates/services/status.html.twig | 8 +- www/templates/stats.html.twig | 2 +- 31 files changed, 343 insertions(+), 286 deletions(-) delete mode 100644 .ideas/birdnet_obs_miner.sh create mode 100755 .ideas/journal.php create mode 100644 .ideas/url_escape.php delete mode 100644 config/species_list.txt create mode 100755 daemon/notify/apprise.sh create mode 100644 daemon/plotter/__init__.py create mode 100755 daemon/plotter/chart.py create mode 100644 daemon/systemd/templates/birdnet_plotter.service create mode 100644 daemon/systemd/templates/birdnet_plotter.timer create mode 100644 www/src/Controller/LogsController.php create mode 100644 www/templates/logs/logs.html.twig diff --git a/.gitignore b/.gitignore index 0ed91ae..08c0732 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ species_list.txt push.sh -config/analyzer.conf \ No newline at end of file +config/*.conf \ No newline at end of file diff --git a/.ideas/birdnet_obs_miner.sh b/.ideas/birdnet_obs_miner.sh deleted file mode 100644 index 997dd16..0000000 --- a/.ideas/birdnet_obs_miner.sh +++ /dev/null @@ -1,35 +0,0 @@ -# !/bin/bash - -# Extract data generated with BirdNET on record to get relevant informations and record data in sqlite - -# Load config file -config_filepath="./config/analyzer.conf" -if [ -f "$config_filepath" ]; then - source "$config_filepath" -else - echo "Config file not found: $config_filepath" - exit 1 -fi - -# Verify needed prerequisites -if [[ -z ${CHUNK_FOLDER} ]]; then - echo "CHUNK_FOLDER is not set" - exit 1 -else - if [[ ! -d "${CHUNK_FOLDER}" ]]; then - echo "CHUNK_FOLDER does not exist: ${CHUNK_FOLDER}" - exit 1 - else - if [[ ! -d "${CHUNK_FOLDER}/out" ]]; then - echo "Output dir does not exist: ${CHUNK_FOLDER}/out" - echo "Cannot mine data" - exit 1 - fi - fi - fi -fi - -function list_all_model_outputs() -{ - -} \ No newline at end of file diff --git a/.ideas/journal.php b/.ideas/journal.php new file mode 100755 index 0000000..c2bad47 --- /dev/null +++ b/.ideas/journal.php @@ -0,0 +1,2 @@ + -Group= +Group= WorkingDirectory= ExecStart=bash ./daemon/birdnet_analyzis.sh Restart=always diff --git a/daemon/systemd/templates/birdnet_miner.service b/daemon/systemd/templates/birdnet_miner.service index ddb8a5d..d24c1f8 100644 --- a/daemon/systemd/templates/birdnet_miner.service +++ b/daemon/systemd/templates/birdnet_miner.service @@ -2,9 +2,9 @@ Description=BirdNET-stream miner service [Service] -Type=oneshot +Type=simple User= -GROUP= +Group= WorkingDirectory= ExecStart=bash ./daemon/birdnet_miner.sh RemainAfterExit=yes diff --git a/daemon/systemd/templates/birdnet_miner.timer b/daemon/systemd/templates/birdnet_miner.timer index fd1ea44..2cf267e 100644 --- a/daemon/systemd/templates/birdnet_miner.timer +++ b/daemon/systemd/templates/birdnet_miner.timer @@ -2,7 +2,7 @@ Description=BirdNET-stream miner Timer [Timer] -OnCalendar=*-*-* *:00 +OnCalendar=*:0/15 Unit=birdnet_miner.service [Install] diff --git a/daemon/systemd/templates/birdnet_plotter.service b/daemon/systemd/templates/birdnet_plotter.service new file mode 100644 index 0000000..d2a3c68 --- /dev/null +++ b/daemon/systemd/templates/birdnet_plotter.service @@ -0,0 +1,12 @@ +[Unit] +Description=BirdNET-stream plotter + +[Service] +User= +Group= +WorkingDirectory= +ExecStart=./.venv/birdnet-stream/bin/python3 ./daemon/plotter/chart.py +Type=simple + +[Install] +WantedBy=multi-user.target diff --git a/daemon/systemd/templates/birdnet_plotter.timer b/daemon/systemd/templates/birdnet_plotter.timer new file mode 100644 index 0000000..a6e7d3a --- /dev/null +++ b/daemon/systemd/templates/birdnet_plotter.timer @@ -0,0 +1,9 @@ +[Unit] +Description=BirdNET-stream plotter timer + +[Timer] +OnCalendar=*:00 +Unit=birdnet_plotter.service + +[Install] +WantedBy=basic.target \ No newline at end of file diff --git a/daemon/systemd/templates/birdnet_recording.service b/daemon/systemd/templates/birdnet_recording.service index 219e4cd..aa15ab0 100644 --- a/daemon/systemd/templates/birdnet_recording.service +++ b/daemon/systemd/templates/birdnet_recording.service @@ -5,7 +5,7 @@ Description=BirdNET-stream recording [Service] User= -Group= +Group= WorkingDirectory= ExecStart=bash ./daemon/birdnet_recording.sh Restart=always diff --git a/install.sh b/install.sh index 8d2a606..0569d1f 100755 --- a/install.sh +++ b/install.sh @@ -64,7 +64,7 @@ install_birdnetstream_services() { DIR=$(pwd) GROUP=$USER debug "Setting up BirdNET stream systemd services" - services="birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_miner.service" + services="birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_miner.service birdnet_plotter.service birdnet_plotter.timer" read -r -a services_array <<<"$services" for service in ${services_array[@]}; do @@ -75,7 +75,7 @@ install_birdnetstream_services() { done done sudo systemctl daemon-reload - sudo systemctl enable --now birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer + sudo systemctl enable --now birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_plotter.timer } install_php8() { diff --git a/requirements.txt b/requirements.txt index 9bc73ef..10591ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,22 +3,33 @@ audioread==2.1.9 certifi==2022.6.15 cffi==1.15.1 charset-normalizer==2.1.0 +cycler==0.11.0 decorator==5.1.1 +fonttools==4.34.4 idna==3.3 joblib==1.1.0 +kiwisolver==1.4.4 librosa==0.9.2 llvmlite==0.39.0 +matplotlib==3.5.3 numba==0.56.0 numpy==1.22.4 packaging==21.3 +pandas==1.4.3 +Pillow==9.2.0 pooch==1.6.0 pycparser==2.21 pyparsing==3.0.9 +python-dateutil==2.8.2 +pytz==2022.2.1 requests==2.28.1 resampy==0.3.1 scikit-learn==1.1.2 scipy==1.9.0 +seaborn==0.11.2 +six==1.16.0 SoundFile==0.10.3.post1 tflite-runtime==2.9.1 threadpoolctl==3.1.0 urllib3==1.26.11 +xmpppy==0.7.1 diff --git a/www/assets/styles/app.css b/www/assets/styles/app.css index 5e0ed5a..1b3de9a 100644 --- a/www/assets/styles/app.css +++ b/www/assets/styles/app.css @@ -58,7 +58,6 @@ header img.logo { main { min-height: 100vh; padding: 5em; - z-index: 0; } footer { @@ -117,7 +116,7 @@ canvas { #statuses .grid { display: grid; grid-template-rows: 1fr 1fr 1fr; - grid-template-columns: 1fr 1fr 1fr; + grid-template-columns: 1fr 1fr 1fr 1fr 1fr; gap: 0.1em; } @@ -130,13 +129,18 @@ canvas { display: inline-block; position: relative; top: 0.4em; + z-index: -1; +} + +.status.inactive.bullet { + background-color: #999; } .status.active.bullet { background-color: #090; } -.status.inactive.bullet { +.status.dead.bullet { background-color: #900; } @@ -151,4 +155,13 @@ canvas { opacity: 100%; background-color: rgba(255, 255, 255, 1); position: relative; +} + +.logs { + background-color: black; + color: white; + font: monospace; + padding: 1em; + overflow-y: scroll; + height: 100%; } \ No newline at end of file diff --git a/www/assets/styles/menu.css b/www/assets/styles/menu.css index 78fcba2..896882f 100644 --- a/www/assets/styles/menu.css +++ b/www/assets/styles/menu.css @@ -3,6 +3,7 @@ nav { top: 0; left: 0; --nav-width: 20em; + z-index: 10; } .toggler{ diff --git a/www/src/Controller/LogsController.php b/www/src/Controller/LogsController.php new file mode 100644 index 0000000..a17d7b1 --- /dev/null +++ b/www/src/Controller/LogsController.php @@ -0,0 +1,38 @@ +allowed_services) as $service) { + $logs .= $this->journal_logs($service); + } + } else if (str_contains($this->allowed_services, $service)) { + $logs .= $this->journal_logs($service); + } else { + return new Response("Service not found", Response::HTTP_BAD_REQUEST); + } + return $this->render('logs/logs.html.twig', [ + 'logs' => $logs + ]); + } + + private function journal_logs($service) + { + $logs = shell_exec("journalctl -u birdnet_recording -n 10"); + return $logs; + } +} diff --git a/www/src/Controller/ServicesController.php b/www/src/Controller/ServicesController.php index dc401bb..bebddec 100644 --- a/www/src/Controller/ServicesController.php +++ b/www/src/Controller/ServicesController.php @@ -5,7 +5,6 @@ use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\Routing\Annotation\Route; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; -use Sensio\Bundle\FrameworkExtraBundle\EventListener\ControllerListener; class ServicesController extends AbstractController { @@ -15,7 +14,7 @@ class ServicesController extends AbstractController /** - * @Route("/service/status", name="service_status") + * @Route("/services/status", name="service_status") */ public function service_status() { $status = array_map(function($service) { @@ -30,15 +29,22 @@ class ServicesController extends AbstractController } /** - * @Route("/service/manage/{action}", name="service_manager") + * @Route("/services/manage/{action}/{service}", name="service_manager") */ - public function service_manage($action, $service) + public function service_manage($action, $service="all") { $error = ""; if (in_array($action, $this->allowed_actions)) { - if (in_array($service, $this->services_available)) { + if ($service == "all") { + foreach ($this->services_available as $service) { + if(($output = $this->manage_systemd_service($action, $service)) != "true") { + $error .= "Error while managing $service service"; + dump($output); + } + } + } else if (in_array($service, $this->services_available)) { if(($output = $this->manage_systemd_service($action, $service)) != "true") { - $error = "Error while managing service"; + $error .= "Error while managing $service service"; dump($output); } } else { @@ -56,18 +62,46 @@ class ServicesController extends AbstractController private function manage_systemd_service($action, $service) { - $command = "./daemon/birdnet_manager.sh ".$action; - $workdir = $this->getParameter("kernel.project_dir") . "/../"; - $command = "cd ".$workdir." && ".$command; - echo $command; + // TODO correct this command (failed with not root user) + $command = "./daemon/birdnet_manager.sh $action birdnet_$service"; + $old_path = getcwd(); + $workdir = $this->getParameter("kernel.project_dir"); + chdir($workdir); $output = shell_exec($command); + dump($output); + chdir($old_path); return $output; } private function systemd_service_status($service) { - $command = "systemctl is-active birdnet_".$service.".service"; - $result = shell_exec($command); - return $result; + $status = array(); + $command = "systemctl is-active birdnet_".$service.".service"; + $output = shell_exec($command); + if (! is_null($output)) + $status["status"] = $output; + else + $status["status"] = "unknown"; + $command = "systemctl is-enabled birdnet_".$service.".service"; + $output = shell_exec($command); + if (! is_null($output)) + $status["enabled"] = $output; + else + $status["enabled"] = "unknown"; + $status["eta"] = $this->systemd_timer_eta($service); + return $status; + } + + private function systemd_timer_eta($service) + { + $eta = ""; + $command = "systemctl list-timers | grep $service.timer | cut -d' ' -f5"; + $output = shell_exec($command); + // dump($output); + if (! is_null($output)) + $eta = $output; + else + $eta = "na"; + return $eta; } } \ No newline at end of file diff --git a/www/templates/index.html.twig b/www/templates/index.html.twig index c81b557..ee382a9 100644 --- a/www/templates/index.html.twig +++ b/www/templates/index.html.twig @@ -1,7 +1,7 @@ {% extends "base.html.twig" %} {% block content %} -

Welcome to BirdNET-stream !

+

{{ "Welcome to BirdNET-stream !" | trans }}

{% include "stats.html.twig" %} {% include "chart.html.twig" %} {% endblock %} \ No newline at end of file diff --git a/www/templates/logs/logs.html.twig b/www/templates/logs/logs.html.twig new file mode 100644 index 0000000..18893fc --- /dev/null +++ b/www/templates/logs/logs.html.twig @@ -0,0 +1,14 @@ +{% extends "base.html.twig" %} + +{% block content %} +

+ {{ "Logs" | trans }} +

+ {% if logs is defined and logs | length > 0 %} + 
+            {{ logs }}
+
+ {% else %} +

{{ "No logs available" | trans }}

+ {% endif %} +{% endblock %} diff --git a/www/templates/menu.html.twig b/www/templates/menu.html.twig index bdf160c..2a7f3b3 100644 --- a/www/templates/menu.html.twig +++ b/www/templates/menu.html.twig @@ -1,49 +1,51 @@ diff --git a/www/templates/services/status.html.twig b/www/templates/services/status.html.twig index 57e0be1..8a51cf3 100644 --- a/www/templates/services/status.html.twig +++ b/www/templates/services/status.html.twig @@ -8,9 +8,11 @@
    {% for service in status %}
  • -
    -
    {{ service["name"] }}
    -
    {{ service["status"] }}
    +
    +
    {{ service.name }}
    +
    {{ service.status.status }}
    +
    {{ service.status.enabled }}
    +
    {{ service.status.eta }}
    • {% endfor %}
diff --git a/www/templates/stats.html.twig b/www/templates/stats.html.twig index bc1625e..534ba26 100644 --- a/www/templates/stats.html.twig +++ b/www/templates/stats.html.twig @@ -1,5 +1,5 @@
-

Quick Stats

+

{{ "Quick Stats" | trans }}

  • {{ "Most recorded species" | trans }}: