From 38fcc7824d082941773e37723be4bbd9a9242cd5 Mon Sep 17 00:00:00 2001 From: Samuel ORTION Date: Sat, 20 Aug 2022 05:22:07 +0200 Subject: [PATCH] Trying to port BirdNET-stream to Docker --- .gitignore | 4 +- CHANGELOG.md | 3 +- CONTRIBUTING.md | 3 + INSTALL.md | 4 +- TODO | 3 + daemon/birdnet_analyzis.sh | 6 +- daemon/birdnet_clean.sh | 80 +++++++++++++++++---------- daemon/database/migrations/.gitignore | 1 + daemon/database/migrations/migrate.sh | 34 ++++++++++++ daemon/database/structure.sql | 3 +- docker-compose.yml | 36 ++++++++++++ docker/all/Dockerfile | 20 +++++++ docker/all/README.md | 27 +++++++++ docker/recording/Dockerfile | 16 ++++++ docker/www/Dockerfile | 0 install.sh | 13 +---- utils/fix_permissions.sh | 25 +++++++++ 17 files changed, 232 insertions(+), 46 deletions(-) create mode 100644 daemon/database/migrations/.gitignore create mode 100755 daemon/database/migrations/migrate.sh create mode 100644 docker-compose.yml create mode 100644 docker/all/Dockerfile create mode 100644 docker/all/README.md create mode 100644 docker/recording/Dockerfile create mode 100644 docker/www/Dockerfile create mode 100644 utils/fix_permissions.sh diff --git a/.gitignore b/.gitignore index 08c0732..3ce8b9c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ species_list.txt push.sh -config/*.conf \ No newline at end of file +config/*.conf + +.vscode/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 3221104..c87fbde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,5 +9,6 @@ - Extracts BirdNET bird contacts into SQL database - Add birdnet_stream icecast audio streaming and live spectrogram service https://birdnet/spectro - Add /today/species and /today/{date}/species/{id} endpoints -- Add records deletion button +- Add records deletion button and /records/delete endpoint as well as bulk deletion (select all button on /today/species/{id} endpoint) - Add systemd status page /status +- Add i18n for webapp (not species name), en|fr only for the moment diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c16cfc..6819844 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,9 @@ BirdNET-stream webapp is written in PHP Symfony. The i18n files are stored in th Any help is welcome to translate the webapp into your language. +Add your language code into [./www/bin/translate.sh](./www/bin/translate.sh) and run it to update the translation files. + +Then, edit generated files in [./www/translations](./www/translations). ## Filing a pull request diff --git a/INSTALL.md b/INSTALL.md index fa5cda5..1100159 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ This guide allow you to install BirdNET-stream step by step on your debian based For a one-liner installation, you can use the following command: ```bash -curl -sL https://raw.githubusercontent.com/birdnet-stream/birdnet-stream/master/install.sh | bash +curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh | bash ``` ## Requirements @@ -34,7 +34,7 @@ sudo apt-get install ffmpeg ### Clone BirdNET-stream repository ```bash -git clone --recurse-submodules https://forge.chapril.org/UncleSamulus/BirdNET-stream.git +git clone --recurse-submodules https://github.com/UncleSamulus/BirdNET-stream.git ``` ### Setup python virtualenv and packages diff --git a/TODO b/TODO index 82ededb..d9235ab 100644 --- a/TODO +++ b/TODO @@ -1,2 +1,5 @@ - Fix clean script - Fix service manager +- Add docker support +- Species i18n +- File purge policy \ No newline at end of file diff --git a/daemon/birdnet_analyzis.sh b/daemon/birdnet_analyzis.sh index a2626ad..3986134 100755 --- a/daemon/birdnet_analyzis.sh +++ b/daemon/birdnet_analyzis.sh @@ -82,7 +82,11 @@ analyze_chunk() { # Perform audio chunk analysis on all recorded chunks analyze_chunks() { for chunk_name in $(get_chunk_list); do - analyze_chunk $chunk_name + if [[ -f "${CHUNK_FOLDER}/out/$chunk_name.d/model.out.csv" ]]; then + debug "Skipping $chunk_name, as it has already been analyzed" + else + analyze_chunk $chunk_name + fi chunk_path="${CHUNK_FOLDER}/in/$chunk_name" mv $chunk_path "${CHUNK_FOLDER}/out/$chunk_name" done diff --git a/daemon/birdnet_clean.sh b/daemon/birdnet_clean.sh index 1094ebd..c9f5639 100755 --- a/daemon/birdnet_clean.sh +++ b/daemon/birdnet_clean.sh @@ -3,6 +3,16 @@ ## Clean up var folder from useless files ## +set -e +# set -x + +DEBUG=${DEBUG:-1} +debug() { + if [[ $DEBUG -eq 1 ]]; then + echo "$1" + fi +} + config_filepath="./config/analyzer.conf" if [ -f "$config_filepath" ]; then @@ -20,51 +30,61 @@ wav2dir_name() { # Clean out folder from empty audio clean() { - rm -rf $(junk) + for item in $(junk); do + debug "Removing: $item" + rm -rf "$CHUNK_FOLDER/out/$item" + done + empty_audios=$(find "$CHUNK_FOLDER/in" -type f -size 0) + for item in $empty_audios; do + rm -rf "$item" + done } -# Check if string contains string -mem() { - string=$2 - substring=$1 - if [[ "$string" == *"$substring"* ]]; then - echo "true" - else - echo "false" - fi +dryclean() { + debug "Dry run mode" + debug "Script will remove the following files:" + for item in $(junk); do + debug "$item" + done + empty_audios=$(find "$CHUNK_FOLDER/in" -type f -size 0) + for item in $empty_audios; do + echo "$item" + done } # Get list of junk files junk() { # Get all empty files from treatement folder - find "${CHUNK_FOLDER}/out" -type f -name '*.wav' -size 0 + junk=$(find "${CHUNK_FOLDER}/out" -type f -name '*.wav' -size 0) for file in $junk; do folder=$(wav2dir_name "$file") if [[ -d $folder ]]; then junk="$junk $folder" fi done - # Get all empty files from record folder - junk=$(find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0) # Get all empty treatment directories - junk="$junk $(find ${CHUNK_FOLDER}/out -type d -empty)" - # Get all empty record directories - treatement_folder=$(find -wholename "${CHUNK_FOLDER}/out/*" -type d ! -empty) - if [[ ! -z ${treatement_folder} ]]; then - for folder in $treatement_folder; do - echo $folder - if [[ ! $(mem $folder $junk) = "true" ]] && $(no_bird_in_model_output $folder); then - junk="$junk $folder" - fi - done - fi + junk="$junk $(find ${CHUNK_FOLDER}/out/* -type d -empty)" + # Get all no birdcontact directories + treatement_folders=$(find ${CHUNK_FOLDER}/out/* -type d ! -empty) + for folder in $treatement_folders; do + folder_basename=$(basename "$folder") + if [[ $(no_bird_in_model_output $folder_basename) = "true" ]]; then + # Add model output file to junk list + junk="$junk $folder_basename/model.out.csv" + junk="$junk $folder_basename" + fi + done echo "$junk" } no_bird_in_model_output() { folder=$1 - output="${folder}/model.out.csv" - lines=$(wc -l < "$output") + output="$CHUNK_FOLDER/out/$folder/model.out.csv" + if [[ -f $output ]]; then + lines=$(wc -l <"$output") + else + lines=0 + fi if [[ $lines -eq 1 ]]; then echo "true" else @@ -72,8 +92,8 @@ no_bird_in_model_output() { fi } -main() { +if [[ $1 = "dry" ]]; then + dryclean +else clean -} - -main \ No newline at end of file +fi diff --git a/daemon/database/migrations/.gitignore b/daemon/database/migrations/.gitignore new file mode 100644 index 0000000..945c9b4 --- /dev/null +++ b/daemon/database/migrations/.gitignore @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/daemon/database/migrations/migrate.sh b/daemon/database/migrations/migrate.sh new file mode 100755 index 0000000..615f344 --- /dev/null +++ b/daemon/database/migrations/migrate.sh @@ -0,0 +1,34 @@ +#! /usr/bin/bash + +DEBUG=${DEBUG:-1} +debug() { + if [ "$DEBUG" -eq 1 ]; then + echo "$1" + fi +} + +# Load config file +config_filepath="./config/analyzer.conf" + +if [ -f "$config_filepath" ]; then + source "$config_filepath" +else + echo "Config file not found: $config_filepath" + exit 1 +fi + +# Check if database location is specified +if [ -z "$DATABASE" ]; then + echo "DATABASE location not specified" + echo "Defaults to ./var/db.sqlite" + DATABASE="./var/db.sqlite" +fi + +if [[ ! -f "$DATABASE" ]]; then + echo "Database file not found: $DATABASE" + exit 1 +fi + +source ./daemon/database/scripts/database.sh + +sqlite3 "$DATABASE" "ALTER TABLE observation ADD COLUMN verified BOOLEAN CHECK (verified IN (0, 1)) DEFAULT 0;" \ No newline at end of file diff --git a/daemon/database/structure.sql b/daemon/database/structure.sql index 2397785..23a288f 100644 --- a/daemon/database/structure.sql +++ b/daemon/database/structure.sql @@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS observation ( `date` TEXT NOT NULL, `notes` TEXT, `confidence` REAL NOT NULL, + `verified` BOOLEAN NOT NULL CHECK (`verified` IN (0, 1)) DEFAULT 0, FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id), FOREIGN KEY(location_id) REFERENCES location(location_id) -); \ No newline at end of file +); diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..fb0afef --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3.8' + +networks: + birdnet_network: + +services: + # database: + # container_name: birdnet_database + # image: + + php: + container_name: birdnet_php + image: php:8.1-fpm + ports: + - "${PHP_FPM_PORT:-9001}:9000" + + + nginx: + container_name: birdnet_nginx + build: + context: ./docker/ + environment: + SERVER_NAME: ${SERVER_NAME:-birdnet.local} + PHP_FPM_PORT: ${PHP_FPM_PORT:-9001} + restart: unless-stopped + volumes: + - ./www:/var/www/birdnet/ + - ./www/nginx.conf:/etc/nginx/conf.d/birdnet.conf + ports: + - "81:80" + dependends_on: + - php + + birdnet: + container_name: birdnet_analyzer + image: \ No newline at end of file diff --git a/docker/all/Dockerfile b/docker/all/Dockerfile new file mode 100644 index 0000000..d39f512 --- /dev/null +++ b/docker/all/Dockerfile @@ -0,0 +1,20 @@ +# All in One BirdNET docker image +FROM debian:bullseye + +ENV REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git} +# DEBUG defaults to 1 for descriptive DEBUG logs, 0 for error logs only +ENV DEBUG=${DEBUG:-1} +RUN useradd birdnet +WORKDIR /home/birdnet + +# Upgrade system +RUN apt-get update && apt-get upgrade -y + +# Install curl +RUN apt-get install -y \ + curl \ + sudo + +RUN curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/master/install.sh | bash + +USER birdnet diff --git a/docker/all/README.md b/docker/all/README.md new file mode 100644 index 0000000..1805f9f --- /dev/null +++ b/docker/all/README.md @@ -0,0 +1,27 @@ +# All in One Docker Container for BirdNET-stream application + +## Requirements + +- docker + +## Quick start + +```bash +git clone https://github.com/UncleSamulus/BirdNET-stream.git +cd ./BirdNET-stream/docker/all +docker build -t "birdnet_all:latest" . +``` + +If `docker` command does not work because of unsufficient permissions, you could add your user to `docker` group: + +```bash +sudo usermod -aG docker $USER +``` + +Then logout, reconnect and try again. + +Then, docker container should be run this way: + +```bash +docker run -it birdnet_all --restart unless-stopped +``` \ No newline at end of file diff --git a/docker/recording/Dockerfile b/docker/recording/Dockerfile new file mode 100644 index 0000000..5f1d603 --- /dev/null +++ b/docker/recording/Dockerfile @@ -0,0 +1,16 @@ +# Recording container for BirdNET-stream +# Reference: https://leimao.github.io/blog/Docker-Container-Audio/ + +FROM debian:bullseye + +ENV DEBIAN_FRONTEND noninteractive + +# Install packages dependencies + +RUN apt-get update && \ + apt-get install apt-utils \ + && apt-get install -y --no-install-recommends \ + libasound2 \ + alsa-utils \ + libsndfile1-dev \ + && apt-get clean diff --git a/docker/www/Dockerfile b/docker/www/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/install.sh b/install.sh index 0569d1f..f1ef391 100755 --- a/install.sh +++ b/install.sh @@ -1,19 +1,12 @@ #! /usr/bin/env bash +# Standard Installation Script for BirdNET-stream for Debian Based Linux distros # set -x set -e DEBUG=${DEBUG:-0} -# Standard Installation Script for BirdNET-stream for Debian Based Linux distros - REQUIREMENTS="git ffmpeg python3-pip python3-dev" -REPOSITORY="https://github.com/UncleSamulus/BirdNET-stream.git" - -# Update system -update() { - sudo apt-get update - sudo apt-get upgrade -y -} +REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git} debug() { if [ $DEBUG -eq 1 ]; then @@ -32,7 +25,7 @@ install_requirements() { done if [ -n "$missing_requirements" ]; then debug "Installing missing requirements: $missing_requirements" - sudo apt-get install $missing_requirements + sudo apt-get install -y $missing_requirements fi } diff --git a/utils/fix_permissions.sh b/utils/fix_permissions.sh new file mode 100644 index 0000000..514cfca --- /dev/null +++ b/utils/fix_permissions.sh @@ -0,0 +1,25 @@ +#! /usr/bin/env bash + +set -e + +DEBUG=${DEBUG:-1} +debug() { + if [ $DEBUG -eq 1 ]; then + echo "$1" + fi +} + +config_filepath="./config/analyzer.conf" + +if [ -f "$config_filepath" ]; then + source "$config_filepath" +else + echo "Config file not found: $config_filepath" + exit 1 +fi + + +GROUP=birdnet + +sudo chown -R $USER:$GROUP $CHUNK_FOLDER +sudo chmod -R 775 $CHUNK_FOLDER \ No newline at end of file