Trying to port BirdNET-stream to Docker

This commit is contained in:
Samuel Ortion 2022-08-20 05:22:07 +02:00
parent 0be5df07a7
commit 38fcc7824d
17 changed files with 232 additions and 46 deletions

2
.gitignore vendored
View File

@ -9,3 +9,5 @@ species_list.txt
push.sh
config/*.conf
.vscode/

View File

@ -9,5 +9,6 @@
- Extracts BirdNET bird contacts into SQL database
- Add birdnet_stream icecast audio streaming and live spectrogram service https://birdnet/spectro
- Add /today/species and /today/{date}/species/{id} endpoints
- Add records deletion button
- Add records deletion button and /records/delete endpoint as well as bulk deletion (select all button on /today/species/{id} endpoint)
- Add systemd status page /status
- Add i18n for webapp (not species name), en|fr only for the moment

View File

@ -22,6 +22,9 @@ BirdNET-stream webapp is written in PHP Symfony. The i18n files are stored in th
Any help is welcome to translate the webapp into your language.
Add your language code into [./www/bin/translate.sh](./www/bin/translate.sh) and run it to update the translation files.
Then, edit generated files in [./www/translations](./www/translations).
## Filing a pull request

View File

@ -5,7 +5,7 @@ This guide allow you to install BirdNET-stream step by step on your debian based
For a one-liner installation, you can use the following command:
```bash
curl -sL https://raw.githubusercontent.com/birdnet-stream/birdnet-stream/master/install.sh | bash
curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh | bash
```
## Requirements
@ -34,7 +34,7 @@ sudo apt-get install ffmpeg
### Clone BirdNET-stream repository
```bash
git clone --recurse-submodules https://forge.chapril.org/UncleSamulus/BirdNET-stream.git
git clone --recurse-submodules https://github.com/UncleSamulus/BirdNET-stream.git
```
### Setup python virtualenv and packages

3
TODO
View File

@ -1,2 +1,5 @@
- Fix clean script
- Fix service manager
- Add docker support
- Species i18n
- File purge policy

View File

@ -82,7 +82,11 @@ analyze_chunk() {
# Perform audio chunk analysis on all recorded chunks
analyze_chunks() {
for chunk_name in $(get_chunk_list); do
if [[ -f "${CHUNK_FOLDER}/out/$chunk_name.d/model.out.csv" ]]; then
debug "Skipping $chunk_name, as it has already been analyzed"
else
analyze_chunk $chunk_name
fi
chunk_path="${CHUNK_FOLDER}/in/$chunk_name"
mv $chunk_path "${CHUNK_FOLDER}/out/$chunk_name"
done

View File

@ -3,6 +3,16 @@
## Clean up var folder from useless files
##
set -e
# set -x
DEBUG=${DEBUG:-1}
debug() {
if [[ $DEBUG -eq 1 ]]; then
echo "$1"
fi
}
config_filepath="./config/analyzer.conf"
if [ -f "$config_filepath" ]; then
@ -20,51 +30,61 @@ wav2dir_name() {
# Clean out folder from empty audio
clean() {
rm -rf $(junk)
for item in $(junk); do
debug "Removing: $item"
rm -rf "$CHUNK_FOLDER/out/$item"
done
empty_audios=$(find "$CHUNK_FOLDER/in" -type f -size 0)
for item in $empty_audios; do
rm -rf "$item"
done
}
# Check if string contains string
mem() {
string=$2
substring=$1
if [[ "$string" == *"$substring"* ]]; then
echo "true"
else
echo "false"
fi
dryclean() {
debug "Dry run mode"
debug "Script will remove the following files:"
for item in $(junk); do
debug "$item"
done
empty_audios=$(find "$CHUNK_FOLDER/in" -type f -size 0)
for item in $empty_audios; do
echo "$item"
done
}
# Get list of junk files
junk() {
# Get all empty files from treatement folder
find "${CHUNK_FOLDER}/out" -type f -name '*.wav' -size 0
junk=$(find "${CHUNK_FOLDER}/out" -type f -name '*.wav' -size 0)
for file in $junk; do
folder=$(wav2dir_name "$file")
if [[ -d $folder ]]; then
junk="$junk $folder"
fi
done
# Get all empty files from record folder
junk=$(find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0)
# Get all empty treatment directories
junk="$junk $(find ${CHUNK_FOLDER}/out -type d -empty)"
# Get all empty record directories
treatement_folder=$(find -wholename "${CHUNK_FOLDER}/out/*" -type d ! -empty)
if [[ ! -z ${treatement_folder} ]]; then
for folder in $treatement_folder; do
echo $folder
if [[ ! $(mem $folder $junk) = "true" ]] && $(no_bird_in_model_output $folder); then
junk="$junk $folder"
junk="$junk $(find ${CHUNK_FOLDER}/out/* -type d -empty)"
# Get all no birdcontact directories
treatement_folders=$(find ${CHUNK_FOLDER}/out/* -type d ! -empty)
for folder in $treatement_folders; do
folder_basename=$(basename "$folder")
if [[ $(no_bird_in_model_output $folder_basename) = "true" ]]; then
# Add model output file to junk list
junk="$junk $folder_basename/model.out.csv"
junk="$junk $folder_basename"
fi
done
fi
echo "$junk"
}
no_bird_in_model_output() {
folder=$1
output="${folder}/model.out.csv"
output="$CHUNK_FOLDER/out/$folder/model.out.csv"
if [[ -f $output ]]; then
lines=$(wc -l <"$output")
else
lines=0
fi
if [[ $lines -eq 1 ]]; then
echo "true"
else
@ -72,8 +92,8 @@ no_bird_in_model_output() {
fi
}
main() {
if [[ $1 = "dry" ]]; then
dryclean
else
clean
}
main
fi

1
daemon/database/migrations/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.

View File

@ -0,0 +1,34 @@
#! /usr/bin/bash
DEBUG=${DEBUG:-1}
debug() {
if [ "$DEBUG" -eq 1 ]; then
echo "$1"
fi
}
# Load config file
config_filepath="./config/analyzer.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"
else
echo "Config file not found: $config_filepath"
exit 1
fi
# Check if database location is specified
if [ -z "$DATABASE" ]; then
echo "DATABASE location not specified"
echo "Defaults to ./var/db.sqlite"
DATABASE="./var/db.sqlite"
fi
if [[ ! -f "$DATABASE" ]]; then
echo "Database file not found: $DATABASE"
exit 1
fi
source ./daemon/database/scripts/database.sh
sqlite3 "$DATABASE" "ALTER TABLE observation ADD COLUMN verified BOOLEAN CHECK (verified IN (0, 1)) DEFAULT 0;"

View File

@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS observation (
`date` TEXT NOT NULL,
`notes` TEXT,
`confidence` REAL NOT NULL,
`verified` BOOLEAN NOT NULL CHECK (`verified` IN (0, 1)) DEFAULT 0,
FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id),
FOREIGN KEY(location_id) REFERENCES location(location_id)
);

36
docker-compose.yml Normal file
View File

@ -0,0 +1,36 @@
version: '3.8'
networks:
birdnet_network:
services:
# database:
# container_name: birdnet_database
# image:
php:
container_name: birdnet_php
image: php:8.1-fpm
ports:
- "${PHP_FPM_PORT:-9001}:9000"
nginx:
container_name: birdnet_nginx
build:
context: ./docker/
environment:
SERVER_NAME: ${SERVER_NAME:-birdnet.local}
PHP_FPM_PORT: ${PHP_FPM_PORT:-9001}
restart: unless-stopped
volumes:
- ./www:/var/www/birdnet/
- ./www/nginx.conf:/etc/nginx/conf.d/birdnet.conf
ports:
- "81:80"
dependends_on:
- php
birdnet:
container_name: birdnet_analyzer
image:

20
docker/all/Dockerfile Normal file
View File

@ -0,0 +1,20 @@
# All in One BirdNET docker image
FROM debian:bullseye
ENV REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git}
# DEBUG defaults to 1 for descriptive DEBUG logs, 0 for error logs only
ENV DEBUG=${DEBUG:-1}
RUN useradd birdnet
WORKDIR /home/birdnet
# Upgrade system
RUN apt-get update && apt-get upgrade -y
# Install curl
RUN apt-get install -y \
curl \
sudo
RUN curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/master/install.sh | bash
USER birdnet

27
docker/all/README.md Normal file
View File

@ -0,0 +1,27 @@
# All in One Docker Container for BirdNET-stream application
## Requirements
- docker
## Quick start
```bash
git clone https://github.com/UncleSamulus/BirdNET-stream.git
cd ./BirdNET-stream/docker/all
docker build -t "birdnet_all:latest" .
```
If `docker` command does not work because of unsufficient permissions, you could add your user to `docker` group:
```bash
sudo usermod -aG docker $USER
```
Then logout, reconnect and try again.
Then, docker container should be run this way:
```bash
docker run -it birdnet_all --restart unless-stopped
```

View File

@ -0,0 +1,16 @@
# Recording container for BirdNET-stream
# Reference: https://leimao.github.io/blog/Docker-Container-Audio/
FROM debian:bullseye
ENV DEBIAN_FRONTEND noninteractive
# Install packages dependencies
RUN apt-get update && \
apt-get install apt-utils \
&& apt-get install -y --no-install-recommends \
libasound2 \
alsa-utils \
libsndfile1-dev \
&& apt-get clean

0
docker/www/Dockerfile Normal file
View File

View File

@ -1,19 +1,12 @@
#! /usr/bin/env bash
# Standard Installation Script for BirdNET-stream for Debian Based Linux distros
# set -x
set -e
DEBUG=${DEBUG:-0}
# Standard Installation Script for BirdNET-stream for Debian Based Linux distros
REQUIREMENTS="git ffmpeg python3-pip python3-dev"
REPOSITORY="https://github.com/UncleSamulus/BirdNET-stream.git"
# Update system
update() {
sudo apt-get update
sudo apt-get upgrade -y
}
REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git}
debug() {
if [ $DEBUG -eq 1 ]; then
@ -32,7 +25,7 @@ install_requirements() {
done
if [ -n "$missing_requirements" ]; then
debug "Installing missing requirements: $missing_requirements"
sudo apt-get install $missing_requirements
sudo apt-get install -y $missing_requirements
fi
}

25
utils/fix_permissions.sh Normal file
View File

@ -0,0 +1,25 @@
#! /usr/bin/env bash
set -e
DEBUG=${DEBUG:-1}
debug() {
if [ $DEBUG -eq 1 ]; then
echo "$1"
fi
}
config_filepath="./config/analyzer.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"
else
echo "Config file not found: $config_filepath"
exit 1
fi
GROUP=birdnet
sudo chown -R $USER:$GROUP $CHUNK_FOLDER
sudo chmod -R 775 $CHUNK_FOLDER