Merge pull request 'dev' (#1) from dev into main

Reviewed-on: UncleSamulus/BirdNET-stream#1
This commit is contained in:
Samuel Ortion 2022-08-25 06:02:34 +02:00
commit d6763f6e54
60 changed files with 4380 additions and 3347 deletions

7
.dockerignore Normal file
View File

@ -0,0 +1,7 @@
/var
/.venv
/.github
/.ideas
/media
/daemon/systemd
/analyzer

1
.env
View File

@ -1 +0,0 @@
CUDA_VISIBLE_DEVICES=""

12
.env.example Normal file
View File

@ -0,0 +1,12 @@
# BirdNET-Analyzer environment
CUDA_VISIBLE_DEVICES=""
# BirdNET-stream environment
DATABASE_USER="birdnet"
DATABASE_PASSWORD="secret" # change this
DATABASE_PORT="3306"
DATABASE_ROOT_PASSWORD="secret" # change this
RECORDS_DIR="/media/data/birdnet/records"
CHARTS_DIR="/media/data/birdnet/charts"
SERVER_NAME="birdnet.local"

12
.gitignore vendored
View File

@ -1,13 +1,17 @@
var/
/.venv/
/analyzer/
.env
/.env
/.env.local
!.env.local.example
!.env.example
species_list.txt
push.sh
config/*.conf
/config/*.conf
!/config/*.conf.example
/config/stations/*.conf
!/config/stations/*.conf.example
.vscode/

133
.ideas/birdnet_archive.sh Executable file
View File

@ -0,0 +1,133 @@
#! /usr/bin/env bash
# Compress wav to flac and archive them as zip
# Requires: tar, gzip, ffmpeg
set -e
DEBUG=${DEBUG:-0}
debug() {
[[ $DEBUG -eq 1 ]] && echo "$@"
}
error() {
echo 1>&2 "$@"
}
audio_compress() {
local filepath
filepath="$1"
if [[ "$DRY" -eq 1 ]]; then
debug "Would compress $filepath to flac"
return 0
else
debug "Compressing $filepath"
ffmpeg -i "$filepath" -acodec flac -compression_level 10 "${filepath%.wav}.flac"
fi
}
all_audio_compress() {
local dir
dir="$1"
debug "Compressing all .wav audio in $dir"
for filepath in "$dir"/*.wav; do
if [[ "$DRY" -eq 1 ]]; then
debug "Would convert $filepath to flac and remove it"
else
audio_compress "$filepath"
debug "Removing $filepath"
rm "$filepath"
fi
done
}
record_datetime() {
source_wav=$1
source_base=$(basename "$source_wav" ".wav")
record_date=$(echo "$source_base" | cut -d"_" -f2)
record_time=$(echo "$source_base" | cut -d"_" -f3)
YYYY=$(echo "$record_date" | cut -c 1-4)
MM=$(echo "$record_date" | cut -c 5-6)
DD=$(echo "$record_date" | cut -c 7-8)
HH=$(echo "$record_time" | cut -c 1-2)
MI=$(echo "$record_time" | cut -c 3-4)
SS=$(echo "$record_time" | cut -c 5-6)
SSS="000"
date="$YYYY-$MM-$DD $HH:$MI:$SS.$SSS"
echo "$date"
}
source_wav() {
model_output_dir="$1"
wav=$(basename "$model_output_dir" | rev | cut --complement -d"." -f1 | rev)
echo "$wav"
}
birdnet_archive_older_than() {
local days
days="$1"
local date
date=$(date +"%Y-%m-%d")
local date_pivot
date_pivot=$(date -d "$date + $days days" +"%Y-%m-%d")
move_records_to_archive "$date_pivot"
zip_archives
}
move_records_to_archive() {
local date
date="$1"
local archives_dir
archives_dir="$2"
archive_path="${ARCHIVE_DIR}/$date"
debug "Moving records from $CHUNK_FOLDER/out to $archives_path"
for filepath in $(find "$CHUNK_FOLDER/out/" -name '*.wav.d'); do
wav=$(source_wav "$filepath")
dir=$(dirname "$filepath")
record_datetime=$(record_datetime "$wav")
if [[ "$record_datetime" == "$date" ]]; then
debug "Moving $filepath to $archive_path"
if [[ ! -d "$archive_path" ]]; then
mkdir -p "$archive_path"
fi
mv "$filepath" "$archive_path"
debug "Moving model output directory to archive"
mv "$dir" "$archive_path/"
debug "Moving wav to archive"
mv "$CHUNK_FOLDER/out/$wav" "$archive_path/"
fi
done
}
zip_archives() {
debug "Zipping archives in ${ARCHIVE_DIR}"
for archive_path in $(find "${ARCHIVE_DIR}" -type d); do
archive_name="birdnet_$(basename "$archive_path" | tr '-' '').tar.gz"
if [[ "$DRY" -eq 1 ]]; then
debug "Would zip $archive_path to $archive_name"
else
debug "Zipping $archive_path to $archive_name"
tar -czf "$archive_name" -C "$archive_path" .
debug "Removing temporary archive folder in ${ARCHIVE_DIR}"
rm -rf "$archive_path"
fi
done
}
main() {
config_filepath="./config/birdnet.conf"
[ -f "$config_filepath" ] || {
error "Config file not found: $config_filepath"
exit 1
}
source "$config_filepath"
if [[ -z "CHUNK_FOLDER" ]]; then
error "CHUNK_FOLDER not set in config file"
exit 1
fi
if [[ -z "ARCHIVE_FOLDER" ]]; then
error "ARCHIVE_FOLDER not set in config file"
exit 1
fi
debug "Launch birdnet archive script from $CHUNK_FOLDER to $ARCHIVE_FOLDER"
birdnet_archive_older_than $DAYS_TO_KEEP
}

View File

@ -7,7 +7,7 @@ verbose = False
"""Load config"""
def load_conf():
with open("./config/analyzer.conf", "r") as f:
with open("./config/birdnet.conf", "r") as f:
conf = f.readlines()
res = dict(map(str.strip, sub.split('=', 1)) for sub in conf if '=' in sub)
return res

View File

@ -3,7 +3,7 @@
set -e
# Load config file
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"

View File

@ -1,7 +1,7 @@
#! /usr/bin/env bash
# Load config file
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"

View File

@ -1,6 +1,11 @@
# Changelog
## v0.0.1-alpha
## v0.0.1-rc
- Add docker compose port
- Improve install script
- Add base uninstall script (need deeper work)
- Add ttyd for systemd logging
## v0.0.1-rc (2022-08-18)
- Integrate BirdNET-Analyzer as submodule
- Add birdnet_recording service
@ -11,4 +16,4 @@
- Add /today/species and /today/{date}/species/{id} endpoints
- Add records deletion button and /records/delete endpoint as well as bulk deletion (select all button on /today/species/{id} endpoint)
- Add systemd status page /status
- Add i18n for webapp (not species name), en|fr only for the moment
- Add i18n for webapp (not species name), en|fr only for the moment

View File

@ -8,6 +8,18 @@ For a one-liner installation, you can use the following command:
curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh | bash
```
For debug purposes, you can use the following command, it will log the installation steps to the console:
```bash
DEBUG=1 ./install.sh
```
If you need to use a specific branch (e.g. dev), you can use the following command:
```bash
BRANCH=dev ./install.sh
```
## Requirements
- git
@ -21,7 +33,7 @@ curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/inst
```bash
sudo apt-get update
sudo apt-get install python3-dev python3-pip
sudo apt-get install python3-dev python3-pip python3-venv
sudo pip3 install --upgrade pip
```
@ -67,17 +79,23 @@ sudo systemctl enable --now birdnet_recording.service birdnet_analyzis.service b
#### Check if services are working
```bash
# Sercices status
sudo systemctl status birdnet_recording.service birdnet_analyzis.service
# Timers status
sudo systemctl status birdnet_miner.timer
# Sercices and timers status
sudo systemctl status birdnet_\*
```
```bash
# BirdNET-stream logs
sudo journalctl -feu {birdnet_recording,birdnet_analyzis}.service
sudo journalctl -feu birdnet_\*
```
#### Enable `loginctl-linger` for the user that runs the servuces
Running:
```bash
loginctl enable-linger
```
This allows to use `/run/user/1000/pulse` to record audio using PulseAudio in birdnet_recording.sh.
## Setup BirdNET-stream symfony webapp
### Install php 8.1
@ -111,7 +129,7 @@ sudo mv /composer.phar /usr/local/bin/composer
```bash
cd www
composer install
composer install --no-dev --prefer-dist --optimize-autoloader
```
### Install nodejs and npm
@ -129,7 +147,7 @@ nvm use 16
```
```bash
sudo dnf install npm
sudo apt-get install npm
```
```bash
@ -170,7 +188,7 @@ Launch and enable icecast:
sudo systemctl enable --now icecast2
```
Adapt `config/analyzer.conf` to this configuration:
Adapt `config/birdnet.conf` to this configuration:
```conf
ICECAST_USER=source
@ -260,3 +278,28 @@ sudo crontab -e
```
(This updates the certicates every first day of the month, feel free to adapt to your needs.)
## Setup ttyd to stream audio to webapp
Change to a dedicated folder, build and install ttyd:
```bash
cd /opt
sudo wget wget https://github.com/tsl0922/ttyd/releases/download/1.7.1/ttyd.x86_64 # Change to your architecture and get last version
sudo mv ttyd.x86_64 ttyd
sudo chmod +x ttyd
```
Set up birdnet_ttyd systemd service to start as a daemon:
```bash
# Copy service template
sudo cp ./daemon/systemd/templates/birdnet_ttyd.service /etc/systemd/system/birdnet_ttyd.service
# Edit template and adapt placeholders
sudo vim /etc/systemd/system/birdnet_ttyd.service
# Enable and start ttyd service
sudo systemctl daemon-reload
sudo systemctl enable --now birdnet_ttyd.service
```
Then go to [https://birdnet.lan/ttyd](https://birdnet.lan/ttyd) and start streaming logs.

View File

@ -8,7 +8,7 @@
## Introduction
BirdNET-stream records sound 24/7 on any Linux computer with a microphone and analyze it using BirdNET algorithm by [**@kahst**](https://github.com/kahst).
BirdNET-stream records sound 24/7 on any Linux computer with a microphone and analyze it using BirdNET algorithm by [**@kahst**](https://github.com/kahst).
Bird contacts are stored in a database and are made accessible in a webapp.
@ -28,19 +28,45 @@ It should work on a Raspberry Pi (or other Single Board Computer) with a USB mic
## Installation
On debian based system, you can install BirdNET-stream with the following command:
> **Warning** BirdNET-stream is in early development, and may not work properly...
<!-- On debian based system, you can install BirdNET-stream with the following command:
```bash
curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh | bash
``` -->
On debian based systems (tested on Debian Bullseye), the following command should allow you to install the base components without too much trouble:
```bash
# Change to your installation directory here, /home/$USER/Documents/BirdNET-stream for instance, or /opt/birdnet-stream, or whatever
cd /path/to/installation/directory
# Download installation script
curl -0 https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh
# Run installation script:
chmod +x ./install.sh
./install.sh
```
For finer control, or to adapt to your system, you can follow the instructions in the [INSTALL.md](./INSTALL.md) file.
I recommend to add `DEBUG=1` before this command to see the installation steps:
```bash
DEBUG=1 ./install.sh
```
To install from a specific git branch, add `BRANCH=<branch>` before the command, for instance:
```bash
BRANCH=dev DEBUG=1 ./install.sh
```
For finer control, or to adapt to your system, you can follow the instructions in the [INSTALL.md](./INSTALL.md) file (it may unfortunatly not be accurate for your system).
## Usage
- BirdNET-stream web application can be accessed on any web browser at [https://birdnet.home](https://birdnet.home), from your local network, or at any other hostname you set in nginx configuration, if your public IP is accessible from the internet.
- See the species detected
- See the species detected
## Acknoledgments
@ -49,4 +75,4 @@ For finer control, or to adapt to your system, you can follow the instructions i
## License
BirdNET-stream is licensed under the GNU General Public License v3.0, see [./LICENSE](./LICENSE) for more details.
BirdNET-stream is licensed under the GNU General Public License v3.0, see [./LICENSE](./LICENSE) for more details.

5
TODO
View File

@ -1,5 +1,4 @@
- Fix clean script
- Fix service manager
- Add docker support
- Species i18n
- File purge policy
- File purge policy
- Add and test RTSP support

View File

@ -16,8 +16,7 @@ CHUNK_FOLDER="./var/chunks"
AUDIO_DEVICE="default"
# Virtual env for BirdNET AI with required packages
PYTHON_VENV="./.venv/birdnet-stream"
WORKDIR="/home/$USER/BirdNET-stream"
# Database location
DATABASE="./var/db.sqlite"
DAEMON_USER="birdnet"
DAEMON_PASSWORD="secret"
# DATABASE="mysql://birdnet:secret@localhost:3306/birdnet_observations" # uncomment and change 'secret' if you want to use a mariadb (mysql) database instea of sqlite

View File

@ -0,0 +1,3 @@
# Config file for a distant recording station
RTSP_URL=rtsp://host:1000/birdnet/stream
STATION_NAME=garden

View File

@ -3,12 +3,10 @@ set -e
DEBUG=${DEBUG:-1}
debug() {
if [ $DEBUG -eq 1 ]; then
echo "$1"
fi
[[ $DEBUG -eq 1 ]] && echo "$@"
}
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"
@ -64,7 +62,9 @@ check_prerequisites() {
# Get array of audio chunks to be processed
get_chunk_list() {
find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0 | sort
chunk_list=($(ls ${CHUNK_FOLDER}/in))
echo "${chunk_list}"
# find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0 | sort
}
# Perform audio chunk analysis on one chunk
@ -75,13 +75,22 @@ analyze_chunk() {
mkdir -p "$output_dir"
date=$(echo $chunk_name | cut -d'_' -f2)
week=$(./daemon/weekof.sh $date)
$PYTHON_EXECUTABLE ./analyzer/analyze.py --i $chunk_path --o "$output_dir/model.out.csv" --lat $LATITUDE --lon $LONGITUDE --week $week --min_conf $CONFIDENCE --threads 4 --rtype csv
if [[ ! -z "${THREADS}" ]]; then
threads="--threads ${THREADS}"
else
threads=""
fi
$PYTHON_EXECUTABLE ./analyzer/analyze.py --i $chunk_path --o "$output_dir/model.out.csv" --lat $LATITUDE --lon $LONGITUDE --week $week --min_conf $CONFIDENCE $threads --rtype csv
debug "Model output written to $output_dir/model.out.csv"
bash ./daemon/birdnet_output_to_sql.sh "$output_dir/model.out.csv"
debug "Dumped to SQL database"
}
# Perform audio chunk analysis on all recorded chunks
analyze_chunks() {
for chunk_name in $(get_chunk_list); do
local chunks
chunks="${1}"
for chunk_name in "${chunks}"; do
if [[ -f "${CHUNK_FOLDER}/out/$chunk_name.d/model.out.csv" ]]; then
debug "Skipping $chunk_name, as it has already been analyzed"
else
@ -98,4 +107,4 @@ check_prerequisites
chunks=$(get_chunk_list)
# Analyze all chunks in working directory
analyze_chunks $chunks
analyze_chunks "$chunks"

View File

@ -1,6 +1,6 @@
#! /usr/bin/env bash
##
## Clean up var folder from useless files
## Clean up var folder from useless files (e.g empty wav, audio with no bird, etc)
##
set -e
@ -13,7 +13,7 @@ debug() {
fi
}
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"

View File

@ -3,7 +3,7 @@
set -e
# set -x
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"

View File

@ -1,33 +1,21 @@
#! /usr/bin/env bash
# Extract observations from a model output folder
# Extract observations from a model output file into SQL database
#
DEBUG=${DEBUG:-1}
set -e
# set -x
DEBUG=${DEBUG:-1}
debug() {
if [ $DEBUG -eq 1 ]; then
echo "$1"
fi
[[ $DEBUG -eq 1 ]] && echo "$@"
}
# Load bash library to deal with BirdNET-stream database
source ./daemon/database/scripts/database.sh
# Load config
source ./config/analyzer.conf
source ./config/birdnet.conf
# Check config
if [[ -z ${CHUNK_FOLDER} ]]; then
echo "CHUNK_FOLDER is not set"
exit 1
else
if [[ ! -d ${CHUNK_FOLDER}/out ]]; then
echo "CHUNK_FOLDER does not exist: ${CHUNK_FOLDER}/out"
echo "Cannot extract observations."
exit 1
fi
fi
if [[ -z ${LATITUDE} ]]; then
echo "LATITUDE is not set"
@ -39,10 +27,6 @@ if [[ -z ${LONGITUDE} ]]; then
exit 1
fi
model_outputs() {
ls ${CHUNK_FOLDER}/out/*/model.out.csv
}
source_wav() {
model_output_path=$1
model_output_dir=$(dirname $model_output_path)
@ -107,13 +91,6 @@ save_observations() {
done
}
main() {
# # Remove all junk observations
# ./daemon/birdnet_clean.sh
# Get model outputs
for model_output in $(model_outputs); do
save_observations $model_output
done
}
model_output_path="$1"
main
save_observations $model_output_path

145
daemon/birdnet_purge.sh Executable file
View File

@ -0,0 +1,145 @@
#! /usr/bin/env bash
# Remove as much as possible as audio files that are not really helpful
#
set -e
# set -x
DEBUG=${DEBUG:-0}
DRY=${DRY:-0}
debug() {
[[ $DEBUG -eq 1 ]] && echo "$@"
}
remove_selected() {
local selected_audios
selected_audios="$1"
for audio in $selected_audios; do
debug "Removing $audio"
rm "$CHUNK_FOLDER/out/$audio"
done
}
remove_audios_older_than() {
local pivot_date=$1
touch -t "$(date -d "$pivot_date" +"%Y%m%d%H%M")" /tmp/birdnet_purge.sh.pivot_date
if [[ $DRY -eq 1 ]]; then
find "$CHUNK_FOLDER/out/" -type f -name '*.wav' -not -newer /tmp/birdnet_purge.sh.pivot_date
else
find "$CHUNK_FOLDER/out/" -type f -name '*.wav' -not -newer /tmp/birdnet_purge.sh.pivot_date -delete
fi
}
# Remove audios containing only excluded species
remove_audios_containing_only_excluded_species() {
local excluded_species
excluded_species=$1
local audios
audios=$(find "$CHUNK_FOLDER/out/" -type f -name '*.wav')
audios=$(only_audios_containing_excluded_species "$audios" "$excluded_species")
if [[ $DRY -eq 1 ]]; then
echo "$audios"
else
remove_selected "$audios"
fi
}
# Filter audio list, keep only those that contains only bird calls of selected exclude list
only_audios_containing_excluded_species() {
local audios
audios=$1
local excluded_species
excluded_species=$2
local selected
selected=""
if [[ -z $excluded_specie ]]; then
echo "No species to exclude"
return 1
fi
for file in $audios; do
if [[ $(contains_only_excluded_species "$file" "$excluded_species") -eq 1 ]]; then
selected="$selected $file"
fi
done
}
# Check whether the audio file contains only excluded species
contains_only_excluded_species() {
local audio
audio=$1
local excluded_species
excluded_species=$2
local flag
flag=1
IFS=$','
local regex
for species in $(get_contacted_species "$audio"); do
regex="$species"
if [[ $excluded_species =~ $regex ]]; then
flag=0
break
fi
done
echo flag
}
# Get all scientific names of species detected by the model
get_contacted_species() {
local audio
audio=$1
local model_output_path
model_output_path="$CHUNK_FOLDER/out/$audio.d/model.out.csv"
observations=$(tail -n +2 < "$model_output_path")
IFS=$'\n'
debug "Observations retrieved from $model_output_path"
local species
local contacted_species
contacted_species=""
for observation in $observations; do
if [[ -z "$observation" ]]; then
continue
fi
species=$(echo "$observation" | cut -d"," -f3)
contacted_species="${contacted_species},${species}"
done
echo "$contacted_species"
}
main() {
debug "Launching birdnet purge script"
local config_path
config_path="./config/birdnet.conf"
if [[ ! -f $config_path ]]; then
echo "Config file $config_path not found"
exit 1
fi
source "$config_path"
if [[ -z ${CHUNK_FOLDER} ]]; then
echo "CHUNK_FOLDER is not set"
exit 1
else
if [[ ! -d ${CHUNK_FOLDER}/out ]]; then
echo "CHUNK_FOLDER does not exist: ${CHUNK_FOLDER}/out"
echo "Cannot clean absent folder."
exit 1
fi
fi
today=$(date +"%Y-%m-%d")
pivot_date=$(date -d "$today - $DAYS_TO_KEEP days" +"%Y-%m-%d")
debug "Recordings older than $pivot_date will be removed"
remove_audios_older_than "$pivot_date"
if [[ -z ${EXCLUDED_SPECIES} ]]; then
echo "No species to exclude"
exit 1
else
if [[ -f ${EXCLUDED_SPECIES} ]]; then
excluded_species=$(cat "${EXCLUDED_SPECIES}")
remove_audios_containing_only_excluded_species "$excluded_species"
else
echo "Excluded species file ${EXCLUDED_SPECIES} not found"
exit 1
fi
fi
}
main

View File

@ -1,8 +1,9 @@
#! /usr/bin/env bash
DEBUG=${DEBUG:-0}
DEBUG=${DEBUG:-1}
export PULSE_RUNTIME_PATH="/run/user/$(id -u)/pulse/"
FFMPEG_OPTIONS="-nostdin -hide_banner -loglevel error -nostats -vn -acodec pcm_s16le -ac 1 -ar 48000"
debug() {
if [ $DEBUG -eq 1 ]; then
@ -23,19 +24,32 @@ record_loop() {
DURATION=$2
debug "New recording loop."
while true; do
record $DEVICE $DURATION
record_device $DEVICE $DURATION
done
}
record() {
record_stream() {
local STREAM=$1
local DURATION=$2
local debug "Recording from $STREAM for $DURATION seconds"
ffmpeg $FFMPEG_OPTIONS -i ${STREAM} -t ${DURATION} file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav
}
record_device() {
DEVICE=$1
DURATION=$2
debug "Recording from $DEVICE for $DURATION seconds"
ffmpeg -nostdin -hide_banner -loglevel error -nostats -f pulse -i ${DEVICE} -t ${DURATION} -vn -acodec pcm_s16le -ac 1 -ar 48000 -af "volume=$RECORDING_AMPLIFY" file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav
local ffmpeg_input
if [[ "$AUDIO_USE_PULSE" = "true" ]]; then
ffmpeg_input="-f pulse -i ${DEVICE}"
else
ffmpeg_input="-f alsa -i ${DEVICE}"
fi
ffmpeg $FFMPEG_OPTIONS $ffmpeg_input -t ${DURATION} -af "volume=$RECORDING_AMPLIFY" file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav
}
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"
@ -48,9 +62,18 @@ check_folder
[ -z $RECORDING_DURATION ] && RECORDING_DURATION=15
if [[ -z $AUDIO_DEVICE ]]; then
echo "AUDIO_DEVICE is not set"
exit 1
if [[ $AUDIO_RECORDING = "true" ]]; then
debug "Recording with on board device"
if [[ -z $AUDIO_DEVICE ]]; then
echo "AUDIO_DEVICE is not set"
exit 1
fi
record_loop $AUDIO_DEVICE $RECORDING_DURATION
fi
record_loop $AUDIO_DEVICE $RECORDING_DURATION
if [[ $AUDIO_STATIONS = "true" ]]; then
for station in $(ls ./config/stations/*.conf); do
source $station
record_stream $STATION_URL $RECORDING_DURATION
done
fi

View File

@ -18,7 +18,7 @@ stream() {
-f mp3 "icecast://source:${ICECAST_PASSWORD}@${ICECAST_HOST}:${ICECAST_PORT}/${ICECAST_MOUNT}" -listen 1
}
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"

View File

@ -8,7 +8,7 @@ debug() {
}
# Load config file
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"

View File

@ -1,7 +1,7 @@
#! /usr/bin/env bash
# Load config file
config_filepath="./config/analyzer.conf"
config_filepath="./config/birdnet.conf"
if [ -f "$config_filepath" ]; then
source "$config_filepath"
@ -17,5 +17,16 @@ if [ -z "$DATABASE" ]; then
DATABASE="./var/db.sqlite"
fi
# Create database according to schema in structure.sql
sqlite3 "$DATABASE" < ./daemon/database/structure.sql
if [[ $DATABASE = "mysql://"* ]]; then
# Split mysql uri into user, password, host, port, and database
MYSQL_ADDRESS=$(echo "$DATABASE" | sed 's/mysql:\/\///g')
MYSQL_CREDENTIALS=$(echo "$MYSQL_ADDRESS" | cut -d@ -f1)
MYSQL_USER=$(echo "$MYSQL_CREDENTIALS" | cut -d: -f1)
MYSQL_PASSWORD=$(echo "$MYSQL_CREDENTIALS" | cut -d: -f2)
MYSQL_HOST=$(echo "$MYSQL_ADDRESS" | cut -d@ -f2 | cut -d: -f1)
MYSQL_PORT=$(echo "$MYSQL_ADDRESS" | cut -d@ -f2 | cut -d: -f2 | cut -d/ -f1)
MYSQL_DATABASE=$(echo "$MYSQL_ADDRESS" | cut -d/ -f2)
mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -h$MYSQL_HOST -P$MYSQL_PORT -D$MYSQL_DATABASE < ./daemon/database/structure-mysql.sql
else
sqlite3 $DATABASE < ./daemon/database/structure-sqlite.sql
fi

View File

@ -1,36 +1,60 @@
#! /usr/bin/env bash
# SQLite library to deal with BirdNET-stream database
# SQLite library to deal with BirdNET-stream observations database
set -e
source ./config/analyzer.conf
source ./config/birdnet.conf
# Create database in case it was not created yet
./daemon/database/scripts/create.sh
DATABASE=${DATABASE:-"./var/db.sqlite"}
# Check if database location is specified
if [ -z "$DATABASE" ]; then
echo "DATABASE location not specified"
echo "Defaults to ./var/db.sqlite"
DATABASE="./var/db.sqlite"
fi
query() {
local stmt
stmt="$1"
if [[ $DATABASE = "mysql://"* ]]; then
# Split mysql uri into user, password, host, port, and database
MYSQL_ADDRESS=$(echo "$DATABASE" | sed 's/mysql:\/\///g')
MYSQL_CREDENTIALS=$(echo "$MYSQL_ADDRESS" | cut -d@ -f1)
MYSQL_USER=$(echo "$MYSQL_CREDENTIALS" | cut -d: -f1)
MYSQL_PASSWORD=$(echo "$MYSQL_CREDENTIALS" | cut -d: -f2)
MYSQL_HOST=$(echo "$MYSQL_ADDRESS" | cut -d@ -f2 | cut -d: -f1)
MYSQL_PORT=$(echo "$MYSQL_ADDRESS" | cut -d@ -f2 | cut -d: -f2 | cut -d/ -f1)
MYSQL_DATABASE=$(echo "$MYSQL_ADDRESS" | cut -d/ -f2)
mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -h$MYSQL_HOST -P$MYSQL_PORT -D$MYSQL_DATABASE -e "$stmt"
else
sqlite3 -cmd ".timeout 1000" "$DATABASE" "$stmt"
fi
}
get_location_id() {
sqlite3 -cmd ".timeout 1000" $DATABASE "SELECT location_id FROM location WHERE latitude=$1 AND longitude=$2"
query "SELECT location_id FROM location WHERE latitude=$1 AND longitude=$2"
}
get_taxon_id() {
sqlite3 -cmd ".timeout 1000" $DATABASE "SELECT taxon_id FROM taxon WHERE scientific_name='$1'"
query "SELECT taxon_id FROM taxon WHERE scientific_name='$1'"
}
insert_taxon() {
sqlite3 -cmd ".timeout 1000" $DATABASE "INSERT INTO taxon (scientific_name, common_name) VALUES (\"$1\", \"$2\")"
query "INSERT INTO taxon (scientific_name, common_name) VALUES (\"$1\", \"$2\")"
}
insert_location() {
sqlite3 -cmd ".timeout 1000" $DATABASE "INSERT INTO location (latitude, longitude) VALUES ($1, $2)"
query "INSERT INTO location (latitude, longitude) VALUES ($1, $2)"
}
insert_observation() {
sqlite3 -cmd ".timeout 1000" $DATABASE "INSERT INTO observation (audio_file, start, end, taxon_id, location_id, confidence, date) VALUES ('$1', '$2', '$3', '$4', '$5', '$6', '$7')"
query "INSERT INTO observation (audio_file, start, end, taxon_id, location_id, confidence, date) VALUES ('$1', '$2', '$3', '$4', '$5', '$6', '$7')"
}
# Check if the observation already exists in the database
observation_exists() {
sqlite3 -cmd ".timeout 1000" $DATABASE "SELECT EXISTS(SELECT observation_id FROM observation WHERE audio_file='$1' AND start='$2' AND end='$3' AND taxon_id='$4' AND location_id='$5')"
}
query "SELECT EXISTS(SELECT observation_id FROM observation WHERE audio_file='$1' AND start='$2' AND end='$3' AND taxon_id='$4' AND location_id='$5')"
}

View File

@ -0,0 +1,31 @@
/** Database structure for BirdNET-stream SQLite*/
/** Taxon table */
CREATE TABLE IF NOT EXISTS taxon (
taxon_id INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT,
scientific_name TEXT NOT NULL,
common_name TEXT NOT NULL
);
/** Location table */
CREATE TABLE IF NOT EXISTS location (
location_id INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT,
latitude REAL NOT NULL,
longitude REAL NOT NULL
);
/** Observation table */
CREATE TABLE IF NOT EXISTS observation (
`observation_id` INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT,
`audio_file` TEXT NOT NULL,
`start` REAL NOT NULL,
`end` REAL NOT NULL,
`taxon_id` INTEGER NOT NULL,
`location_id` INTEGER NOT NULL,
`date` TEXT NOT NULL,
`notes` TEXT,
`confidence` REAL NOT NULL,
`verified` BOOLEAN DEFAULT 0 CHECK (`verified` IN (0, 1)),
FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id),
FOREIGN KEY(location_id) REFERENCES location(location_id)
);

View File

@ -2,21 +2,21 @@
/** Taxon table */
CREATE TABLE IF NOT EXISTS taxon (
taxon_id INTEGER PRIMARY KEY,
taxon_id INTEGER PRIMARY KEY NOT NULL,
scientific_name TEXT NOT NULL,
common_name TEXT NOT NULL
);
/** Location table */
CREATE TABLE IF NOT EXISTS location (
location_id INTEGER PRIMARY KEY,
location_id INTEGER PRIMARY KEY NOT NULL,
latitude REAL NOT NULL,
longitude REAL NOT NULL
);
/** Observation table */
CREATE TABLE IF NOT EXISTS observation (
`observation_id` INTEGER PRIMARY KEY,
`observation_id` INTEGER PRIMARY KEY NOT NULL,
`audio_file` TEXT NOT NULL,
`start` REAL NOT NULL,
`end` REAL NOT NULL,
@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS observation (
`date` TEXT NOT NULL,
`notes` TEXT,
`confidence` REAL NOT NULL,
`verified` BOOLEAN NOT NULL CHECK (`verified` IN (0, 1)) DEFAULT 0,
`verified` BOOLEAN DEFAULT 0 CHECK (`verified` IN (0, 1)),
FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id),
FOREIGN KEY(location_id) REFERENCES location(location_id)
);

View File

@ -1,76 +1,74 @@
#! /usr/bin/env python3
from curses import def_prog_mode
import sqlite3
from xml.sax.handler import feature_external_ges
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm
import seaborn as sns
from datetime import datetime
import os
import glob
CONFIG = {
"readings": 10,
"palette": "Greens",
"db": "./var/db.sqlite",
"date": datetime.now().strftime("%Y-%m-%d"),
"charts_dir": "./var/charts"
}
db = None
def get_database():
global db
if db is None:
db = sqlite3.connect('/home/ortion/Desktop/db.sqlite')
db = sqlite3.connect(CONFIG["db"])
return db
def get_detection_hourly(date):
db = get_database()
df = pd.read_sql_query("""SELECT common_name, date, location_id, confidence
def chart(date):
db = get_database()
df = pd.read_sql_query(f"""SELECT common_name, date, location_id, confidence
FROM observation
INNER JOIN taxon
ON observation.taxon_id = taxon.taxon_id""", db)
ON observation.taxon_id = taxon.taxon_id
WHERE STRFTIME("%Y-%m-%d", `date`) = '{date}'""", db)
df['date'] = pd.to_datetime(df['date'])
df['hour'] = df['date'].dt.hour
df['date'] = df['date'].dt.date
df['date'] = df['date'].astype(str)
df_on_date = df[df['date'] == date]
return df_on_date
top_on_date = (df_on_date['common_name'].value_counts()[:CONFIG['readings']])
if top_on_date.empty:
print("No observations on {}".format(date))
return
else:
print(f"Found observations on {date}")
df_top_on_date = df_on_date[df_on_date['common_name'].isin(top_on_date.index)]
def get_top_species(df, limit=10):
return df['common_name'].value_counts()[:CONFIG['readings']]
def get_top_detections(df, limit=10):
df_top_species = get_top_species(df, limit=limit)
return df[df['common_name'].isin(df_top_species.index)]
def get_frequence_order(df, limit=10):
pd.value_counts(df['common_name']).iloc[:limit]
def presence_chart(date, filename):
df_detections = get_detection_hourly(date)
df_top_detections = get_top_detections(df_detections, limit=CONFIG['readings'])
fig, axs = plt.subplots(1, 2, figsize=(15, 4), gridspec_kw=dict(
width_ratios=[3, 6]))
# Create a figure with 2 subplots
fig, axs = plt.subplots(1, 2, figsize=(20, 5), gridspec_kw=dict(
width_ratios=[2, 6]))
plt.subplots_adjust(left=None, bottom=None, right=None,
top=None, wspace=0, hspace=0)
frequencies_order = get_frequence_order(df_detections, limit=CONFIG["readings"])
# Get species frequencies
frequencies_order = pd.value_counts(df_top_on_date['common_name']).iloc[:CONFIG['readings']].index
# Get min max confidences
confidence_minmax = df_detections.groupby('common_name')['confidence'].max()
confidence_minmax = df_top_on_date.groupby('common_name')['confidence'].max()
confidence_minmax = confidence_minmax.reindex(frequencies_order)
# Norm values for color palette
norm = plt.Normalize(confidence_minmax.values.min(),
confidence_minmax.values.max())
confidence_minmax.values.max())
colors = plt.cm.Greens(norm(confidence_minmax))
plot = sns.countplot(y='common_name', data=df_top_detections, palette=colors, order=frequencies_order, ax=axs[0])
plot = sns.countplot(y='common_name', data=df_top_on_date, palette=colors, order=frequencies_order, ax=axs[0])
plot.set(ylabel=None)
plot.set(xlabel="Detections")
heat = pd.crosstab(df_top_detections['common_name'], df_top_detections['hour'])
heat = pd.crosstab(df_top_on_date['common_name'], df_top_on_date['hour'])
# Order heatmap Birds by frequency of occurrance
heat.index = pd.CategoricalIndex(heat.index, categories=frequencies_order)
heat.sort_index(level=0, inplace=True)
@ -94,27 +92,33 @@ def presence_chart(date, filename):
linewidth=0.5,
linecolor="Grey",
ax=axs[1],
yticklabels=False
)
yticklabels=False)
plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
for _, spine in plot.spines.items():
spine.set_visible(True)
plot.set(ylabel=None)
plot.set(xlabel="Hour of day")
fig.subplots_adjust(top=0.9)
plt.suptitle(f"Top {CONFIG['readings']} species (Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})")
plt.savefig(filename)
plt.suptitle(f"Top {CONFIG['readings']} species on {date}", fontsize=14)
plt.text(15, 11, f"(Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})")
plt.savefig(f"./var/charts/chart_{date}.png", dpi=300)
print(f"Plot for {date} saved.")
plt.close()
def main():
date = datetime.now().strftime('%Y%m%d')
presence_chart(date, f'./var/charts/chart_{date}.png')
# print(get_top_detections(get_detection_hourly(date), limit=10))
if not db is None:
done_charts = glob.glob(f"{CONFIG['charts_dir']}/*.png")
last_modified = max(done_charts, key=os.path.getctime)
last_modified_date = last_modified.split("_")[-1].split(".")[0]
missing_dates = pd.date_range(start=last_modified_date, end=CONFIG['date'], freq='D')
print(missing_dates)
for missing_date in missing_dates:
date = missing_date.strftime("%Y-%m-%d")
chart(date)
chart(CONFIG['date'])
if db is not None:
db.close()
print("Done.")
if __name__ == "__main__":
main()
main()

View File

@ -1,13 +0,0 @@
[Unit]
Description=BirdNET-stream miner service
[Service]
Type=simple
User=<USER>
Group=<GROUP>
WorkingDirectory=<DIR>
ExecStart=bash ./daemon/birdnet_miner.sh
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target

View File

@ -1,9 +0,0 @@
[Unit]
Description=BirdNET-stream miner Timer
[Timer]
OnCalendar=*:0/15
Unit=birdnet_miner.service
[Install]
WantedBy=basic.target

View File

@ -5,7 +5,7 @@ Description=BirdNET-stream plotter
User=<USER>
Group=<GROUP>
WorkingDirectory=<DIR>
ExecStart=./.venv/birdnet-stream/bin/python3 ./daemon/plotter/chart.py
ExecStart=<VENV>/bin/python3 ./daemon/plotter/chart.py
Type=simple
[Install]

View File

@ -0,0 +1,14 @@
[Unit]
Description=BirdNET-stream logs
After=syslog.target
After=network.target
[Service]
User=<USER>
Group=<GROUP>
ExecStart=/opt/ttyd -p 7681 -c birdnet:secret -t disableReconnect=true --readonly journalctl -feu birdnet_\*
Restart=always
Type=simple
[Install]
WantedBy=multi-user.target

View File

@ -1,36 +1,119 @@
version: '3.8'
networks:
birdnet_network:
version: '3.9'
services:
# database:
# container_name: birdnet_database
# image:
# recording:
# container_name: birdnet_recording
# build:
# context: .
# dockerfile: ./docker/recording/Dockerfile
# restart: unless-stopped
# environment:
# - CHUNK_FOLDER=${CHUNK_FOLDER:-/media/birdnet/records}
# volumes:
# - ${RECORDS_DIR:-/media/birdnet/records}:${RECORS_FOLDER:-/media/birdnet/records}
# # Allow container to access to the hosts microphone
# devices:
# - /dev/snd:/dev/snd
# analyzer:
# container_name: birdnet_analyzer
# build:
# context: ./analyzer/
# dockerfile: ./Dockerfile
php:
container_name: birdnet_php
image: php:8.1-fpm
db:
container_name: birdnet_database
image: mariadb:latest
command: mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
ports:
- "${PHP_FPM_PORT:-9001}:9000"
- ${DATABASE_PORT:-3306}:3306
networks:
- birdnet_network
environment:
MYSQL_ROOT_PASSWORD: ${DATABASE_ROOT_PASSWORD:-secret}'
MYSQL_USER: ${DATABASE_USER:-birdnet}
MYSQL_PASSWORD: ${DATABASE_PASSWORD:-secret}
volumes:
- ./docker/database/init:/docker-entrypoint-initdb.d
restart: unless-stopped
php-fpm:
container_name: birdnet_php-fpm
build:
context: .
dockerfile: ./docker/php-fpm/Dockerfile
ports:
- '${PHP_FPM_PORT:-9000}:9000'
networks:
- birdnet_network
environment:
- APP_ENV=${APP_ENV:-prod}
- APP_DEBUG=${APP_DEBUG:-true}
- DATABASE_DEFAULT_URL=mysql://${DATABASE_USER:-birdnet}:${DATABASE_PASSWORD:-secret}@${DATABASE_HOST:-birdnet_database}:${DATABASE_PORT:-3306}/birdnet_default
- DATABASE_OBSERVATIONS_URL=mysql://${DATABASE_USER:-birdnet}:${DATABASE_PASSWORD:-secret}@${DATABASE_HOST:-birdnet_database}:${DATABASE_PORT:-3306}/birdnet_observations
restart: unless-stopped
volumes:
- birdnet_app:${PROJECT_ROOT:-/opt/birdnet}
symfony:
container_name: birdnet_symfony
networks:
- birdnet_network
build:
context: .
dockerfile: ./docker/symfony/Dockerfile
args:
- DATABASE_DEFAULT_URL=mysql://${DATABASE_USER:-birdnet}:${DATABASE_PASSWORD:-secret}@${DATABASE_HOST:-birdnet_database}:${DATABASE_PORT:-3306}/birdnet_default
- DATABASE_OBSERVATIONS_URL=mysql://${DATABASE_USER:-birdnet}:${DATABASE_PASSWORD:-secret}@${DATABASE_HOST:-birdnet_database}:${DATABASE_PORT:-3306}/birdnet_observations
- RECORDS_DIR=/media/birdnet/records
- CHARTS_DIR=/media/birdnet/charts
restart: on-failure
volumes:
- birdnet_app:${PROJECT_ROOT:-/opt/birdnet}
- birdnet_records:${RECORDS_DIR:-/media/birdnet/records}
depends_on:
- db
nginx:
container_name: birdnet_nginx
hostname: ${SERVER_NAME:-birdnet.local}
build:
context: ./docker/
environment:
SERVER_NAME: ${SERVER_NAME:-birdnet.local}
PHP_FPM_PORT: ${PHP_FPM_PORT:-9001}
restart: unless-stopped
volumes:
- ./www:/var/www/birdnet/
- ./www/nginx.conf:/etc/nginx/conf.d/birdnet.conf
context: .
dockerfile: ./docker/nginx/Dockerfile
args:
- SERVER_NAME=${SERVER_NAME:-birnet.local}
- SYMFONY_PUBLIC=/opt/birdnet/www/public
- CHARTS_DIR=/media/birdnet/charts
- RECORDS_DIR=/media/birdnet/records
- PHP_FPM_HOST=birdnet_php-fpm
- PHP_FPM_PORT=9000
ports:
- "81:80"
dependends_on:
- php
- ${HTTP_PORT:-80}:80
- ${HTTPS_PORT:-443}:443
volumes:
- birdnet_app:/opt/birdnet
- birdnet_records:/media/data/records
networks:
birdnet_network:
ipv4_address: ${IP_ADDRESS:-172.25.0.101}
aliases:
- ${SERVER_NAME:-birdnet.local}
restart: unless-stopped
depends_on:
- symfony
- php-fpm
birdnet:
container_name: birdnet_analyzer
image:
networks:
birdnet_network:
driver: bridge
ipam:
config:
- subnet: ${IP_SUBNET:-172.25.0.0/24}
volumes:
birdnet_app:
birdnet_records:
driver_opts:
type: none
device: ${RECORDS_DIR:-/media/data/records}
o: bind

View File

@ -4,17 +4,24 @@ FROM debian:bullseye
ENV REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git}
# DEBUG defaults to 1 for descriptive DEBUG logs, 0 for error logs only
ENV DEBUG=${DEBUG:-1}
RUN useradd birdnet
WORKDIR /home/birdnet
RUN useradd -m -s /bin/bash -G sudo birdnet
USER birdnet
# Upgrade system
RUN apt-get update && apt-get upgrade -y
# Install curl
# Install some dependencies
RUN apt-get install -y \
sudo \
git \
curl \
sudo
bash \
vim \
systemctl
RUN curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/master/install.sh | bash
COPY ./install.sh install.sh
USER birdnet
RUN ./install.sh
EXPOSE 443

View File

@ -9,7 +9,7 @@
```bash
git clone https://github.com/UncleSamulus/BirdNET-stream.git
cd ./BirdNET-stream/docker/all
docker build -t "birdnet_all:latest" .
docker build -t "birdnet_all:latest" -f ./docker/all/Dockerfile .
```
If