Trying to port BirdNET-stream to Docker

This commit is contained in:
Samuel Ortion 2022-08-20 05:22:07 +02:00
parent 0be5df07a7
commit 38fcc7824d
17 changed files with 232 additions and 46 deletions

2
.gitignore vendored
View File

@ -9,3 +9,5 @@ species_list.txt
push.sh push.sh
config/*.conf config/*.conf
.vscode/

View File

@ -9,5 +9,6 @@
- Extracts BirdNET bird contacts into SQL database - Extracts BirdNET bird contacts into SQL database
- Add birdnet_stream icecast audio streaming and live spectrogram service https://birdnet/spectro - Add birdnet_stream icecast audio streaming and live spectrogram service https://birdnet/spectro
- Add /today/species and /today/{date}/species/{id} endpoints - Add /today/species and /today/{date}/species/{id} endpoints
- Add records deletion button - Add records deletion button and /records/delete endpoint as well as bulk deletion (select all button on /today/species/{id} endpoint)
- Add systemd status page /status - Add systemd status page /status
- Add i18n for webapp (not species name), en|fr only for the moment

View File

@ -22,6 +22,9 @@ BirdNET-stream webapp is written in PHP Symfony. The i18n files are stored in th
Any help is welcome to translate the webapp into your language. Any help is welcome to translate the webapp into your language.
Add your language code into [./www/bin/translate.sh](./www/bin/translate.sh) and run it to update the translation files.
Then, edit generated files in [./www/translations](./www/translations).
## Filing a pull request ## Filing a pull request

View File

@ -5,7 +5,7 @@ This guide allow you to install BirdNET-stream step by step on your debian based
For a one-liner installation, you can use the following command: For a one-liner installation, you can use the following command:
```bash ```bash
curl -sL https://raw.githubusercontent.com/birdnet-stream/birdnet-stream/master/install.sh | bash curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh | bash
``` ```
## Requirements ## Requirements
@ -34,7 +34,7 @@ sudo apt-get install ffmpeg
### Clone BirdNET-stream repository ### Clone BirdNET-stream repository
```bash ```bash
git clone --recurse-submodules https://forge.chapril.org/UncleSamulus/BirdNET-stream.git git clone --recurse-submodules https://github.com/UncleSamulus/BirdNET-stream.git
``` ```
### Setup python virtualenv and packages ### Setup python virtualenv and packages

3
TODO
View File

@ -1,2 +1,5 @@
- Fix clean script - Fix clean script
- Fix service manager - Fix service manager
- Add docker support
- Species i18n
- File purge policy

View File

@ -82,7 +82,11 @@ analyze_chunk() {
# Perform audio chunk analysis on all recorded chunks # Perform audio chunk analysis on all recorded chunks
analyze_chunks() { analyze_chunks() {
for chunk_name in $(get_chunk_list); do for chunk_name in $(get_chunk_list); do
analyze_chunk $chunk_name if [[ -f "${CHUNK_FOLDER}/out/$chunk_name.d/model.out.csv" ]]; then
debug "Skipping $chunk_name, as it has already been analyzed"
else
analyze_chunk $chunk_name
fi
chunk_path="${CHUNK_FOLDER}/in/$chunk_name" chunk_path="${CHUNK_FOLDER}/in/$chunk_name"
mv $chunk_path "${CHUNK_FOLDER}/out/$chunk_name" mv $chunk_path "${CHUNK_FOLDER}/out/$chunk_name"
done done

View File

@ -3,6 +3,16 @@
## Clean up var folder from useless files ## Clean up var folder from useless files
## ##
set -e
# set -x
DEBUG=${DEBUG:-1}
debug() {
if [[ $DEBUG -eq 1 ]]; then
echo "$1"
fi
}
config_filepath="./config/analyzer.conf" config_filepath="./config/analyzer.conf"
if [ -f "$config_filepath" ]; then if [ -f "$config_filepath" ]; then
@ -20,51 +30,61 @@ wav2dir_name() {
# Clean out folder from empty audio # Clean out folder from empty audio
clean() { clean() {
rm -rf $(junk) for item in $(junk); do
debug "Removing: $item"
rm -rf "$CHUNK_FOLDER/out/$item"
done
empty_audios=$(find "$CHUNK_FOLDER/in" -type f -size 0)
for item in $empty_audios; do
rm -rf "$item"
done
} }
# Check if string contains string dryclean() {
mem() { debug "Dry run mode"
string=$2 debug "Script will remove the following files:"
substring=$1 for item in $(junk); do
if [[ "$string" == *"$substring"* ]]; then debug "$item"
echo "true" done
else empty_audios=$(find "$CHUNK_FOLDER/in" -type f -size 0)
echo "false" for item in $empty_audios; do
fi echo "$item"
done
} }
# Get list of junk files # Get list of junk files
junk() { junk() {
# Get all empty files from treatement folder # Get all empty files from treatement folder
find "${CHUNK_FOLDER}/out" -type f -name '*.wav' -size 0 junk=$(find "${CHUNK_FOLDER}/out" -type f -name '*.wav' -size 0)
for file in $junk; do for file in $junk; do
folder=$(wav2dir_name "$file") folder=$(wav2dir_name "$file")
if [[ -d $folder ]]; then if [[ -d $folder ]]; then
junk="$junk $folder" junk="$junk $folder"
fi fi
done done
# Get all empty files from record folder
junk=$(find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0)
# Get all empty treatment directories # Get all empty treatment directories
junk="$junk $(find ${CHUNK_FOLDER}/out -type d -empty)" junk="$junk $(find ${CHUNK_FOLDER}/out/* -type d -empty)"
# Get all empty record directories # Get all no birdcontact directories
treatement_folder=$(find -wholename "${CHUNK_FOLDER}/out/*" -type d ! -empty) treatement_folders=$(find ${CHUNK_FOLDER}/out/* -type d ! -empty)
if [[ ! -z ${treatement_folder} ]]; then for folder in $treatement_folders; do
for folder in $treatement_folder; do folder_basename=$(basename "$folder")
echo $folder if [[ $(no_bird_in_model_output $folder_basename) = "true" ]]; then
if [[ ! $(mem $folder $junk) = "true" ]] && $(no_bird_in_model_output $folder); then # Add model output file to junk list
junk="$junk $folder" junk="$junk $folder_basename/model.out.csv"
fi junk="$junk $folder_basename"
done fi
fi done
echo "$junk" echo "$junk"
} }
no_bird_in_model_output() { no_bird_in_model_output() {
folder=$1 folder=$1
output="${folder}/model.out.csv" output="$CHUNK_FOLDER/out/$folder/model.out.csv"
lines=$(wc -l < "$output") if [[ -f $output ]]; then
lines=$(wc -l <"$output")
else
lines=0
fi
if [[ $lines -eq 1 ]]; then if [[ $lines -eq 1 ]]; then
echo "true" echo "true"
else else
@ -72,8 +92,8 @@ no_bird_in_model_output() {
fi fi
} }
main() { if [[ $1 = "dry" ]]; then
dryclean
else
clean clean
} fi
main

1
daemon/database/migrations/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.

View File

@ -0,0 +1,34 @@
#! /usr/bin/bash
DEBUG=${DEBUG:-1}
debug() {
if [ "$DEBUG" -eq 1 ]; then
echo "$1"
fi
}
# Load config file
config_filepath="./config/analyzer.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"
else
echo "Config file not found: $config_filepath"
exit 1
fi
# Check if database location is specified
if [ -z "$DATABASE" ]; then
echo "DATABASE location not specified"
echo "Defaults to ./var/db.sqlite"
DATABASE="./var/db.sqlite"
fi
if [[ ! -f "$DATABASE" ]]; then
echo "Database file not found: $DATABASE"
exit 1
fi
source ./daemon/database/scripts/database.sh
sqlite3 "$DATABASE" "ALTER TABLE observation ADD COLUMN verified BOOLEAN CHECK (verified IN (0, 1)) DEFAULT 0;"

View File

@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS observation (
`date` TEXT NOT NULL, `date` TEXT NOT NULL,
`notes` TEXT, `notes` TEXT,
`confidence` REAL NOT NULL, `confidence` REAL NOT NULL,
`verified` BOOLEAN NOT NULL CHECK (`verified` IN (0, 1)) DEFAULT 0,
FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id), FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id),
FOREIGN KEY(location_id) REFERENCES location(location_id) FOREIGN KEY(location_id) REFERENCES location(location_id)
); );

36
docker-compose.yml Normal file
View File

@ -0,0 +1,36 @@
version: '3.8'
networks:
birdnet_network:
services:
# database:
# container_name: birdnet_database
# image:
php:
container_name: birdnet_php
image: php:8.1-fpm
ports:
- "${PHP_FPM_PORT:-9001}:9000"
nginx:
container_name: birdnet_nginx
build:
context: ./docker/
environment:
SERVER_NAME: ${SERVER_NAME:-birdnet.local}
PHP_FPM_PORT: ${PHP_FPM_PORT:-9001}
restart: unless-stopped
volumes:
- ./www:/var/www/birdnet/
- ./www/nginx.conf:/etc/nginx/conf.d/birdnet.conf
ports:
- "81:80"
dependends_on:
- php
birdnet:
container_name: birdnet_analyzer
image:

20
docker/all/Dockerfile Normal file
View File

@ -0,0 +1,20 @@
# All in One BirdNET docker image
FROM debian:bullseye
ENV REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git}
# DEBUG defaults to 1 for descriptive DEBUG logs, 0 for error logs only
ENV DEBUG=${DEBUG:-1}
RUN useradd birdnet
WORKDIR /home/birdnet
# Upgrade system
RUN apt-get update && apt-get upgrade -y
# Install curl
RUN apt-get install -y \
curl \
sudo
RUN curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/master/install.sh | bash
USER birdnet

27
docker/all/README.md Normal file
View File

@ -0,0 +1,27 @@
# All in One Docker Container for BirdNET-stream application
## Requirements
- docker
## Quick start
```bash
git clone https://github.com/UncleSamulus/BirdNET-stream.git
cd ./BirdNET-stream/docker/all
docker build -t "birdnet_all:latest" .
```
If `docker` command does not work because of unsufficient permissions, you could add your user to `docker` group:
```bash
sudo usermod -aG docker $USER
```
Then logout, reconnect and try again.
Then, docker container should be run this way:
```bash
docker run -it birdnet_all --restart unless-stopped
```

View File

@ -0,0 +1,16 @@
# Recording container for BirdNET-stream
# Reference: https://leimao.github.io/blog/Docker-Container-Audio/
FROM debian:bullseye
ENV DEBIAN_FRONTEND noninteractive
# Install packages dependencies
RUN apt-get update && \
apt-get install apt-utils \
&& apt-get install -y --no-install-recommends \
libasound2 \
alsa-utils \
libsndfile1-dev \
&& apt-get clean

0
docker/www/Dockerfile Normal file
View File

View File

@ -1,19 +1,12 @@
#! /usr/bin/env bash #! /usr/bin/env bash
# Standard Installation Script for BirdNET-stream for Debian Based Linux distros
# set -x # set -x
set -e set -e
DEBUG=${DEBUG:-0} DEBUG=${DEBUG:-0}
# Standard Installation Script for BirdNET-stream for Debian Based Linux distros
REQUIREMENTS="git ffmpeg python3-pip python3-dev" REQUIREMENTS="git ffmpeg python3-pip python3-dev"
REPOSITORY="https://github.com/UncleSamulus/BirdNET-stream.git" REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git}
# Update system
update() {
sudo apt-get update
sudo apt-get upgrade -y
}
debug() { debug() {
if [ $DEBUG -eq 1 ]; then if [ $DEBUG -eq 1 ]; then
@ -32,7 +25,7 @@ install_requirements() {
done done
if [ -n "$missing_requirements" ]; then if [ -n "$missing_requirements" ]; then
debug "Installing missing requirements: $missing_requirements" debug "Installing missing requirements: $missing_requirements"
sudo apt-get install $missing_requirements sudo apt-get install -y $missing_requirements
fi fi
} }

25
utils/fix_permissions.sh Normal file
View File

@ -0,0 +1,25 @@
#! /usr/bin/env bash
set -e
DEBUG=${DEBUG:-1}
debug() {
if [ $DEBUG -eq 1 ]; then
echo "$1"
fi
}
config_filepath="./config/analyzer.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"
else
echo "Config file not found: $config_filepath"
exit 1
fi
GROUP=birdnet
sudo chown -R $USER:$GROUP $CHUNK_FOLDER
sudo chmod -R 775 $CHUNK_FOLDER