Compare commits

...

2 Commits

Author SHA1 Message Date
Quentin Duchemin 68257a9f01
Bump FW to 1.3.0 2023-06-11 22:43:57 +02:00
Quentin Duchemin c6304c8f40
Add Restic/autorestic backups for Funkwhale 2023-06-11 22:03:06 +02:00
16 changed files with 313 additions and 187 deletions

View File

@ -55,3 +55,6 @@
tags:
- docker
- minecraft
- role: restic
tags:
- restic

View File

@ -1,65 +1,67 @@
$ANSIBLE_VAULT;1.1;AES256
35303161333437343431323038663566316162346639373961613238613030636136663161393565
6631633661303935373638656430316535633864643863630a373730326137373031336536336261
32373563333034336262326330623764666434633263626339376432306534636237386432313135
3866376666333962610a333636636664626133633535663331613765643963326164353765363661
38366465306236373137663765636566306437313438353466333731393565323335666463663337
37313735343532303639363831313332653564326364633964323966653866396139343630356666
66323639316261333733343961373061393166336364623536316438363038356438366133343635
38313866636132616238636263633730346633336165303631643166336233353433333961396430
38653234353866383863633037323738623936363333633335333563306639393330363734356531
33623465363862383439306166313662616232343932356461313635326234343339656636636230
63353961613531313164336131383761373633353864303735313037343665613638323264343832
34366663626662643838303036353365626362623030623536653534303865393636623365633534
38356565373361353733366562343937643937626337343966303031373337346232633662633534
32373164636639333635376438636635346164376264323465373136633934383236363661613930
61393862393065363835303639626539383936343835623365343964343763316539333837643031
33643731376333363461666136656130643934383830326534393330363434633165333934323562
35636464376439343062616336633339643031346462386430623965633964623363333234366663
34623939383164373762376235323166393766306462643162613239363631653634373133343963
39313236323965623139643961363830316361393662636431636438303564306639363737333131
62323266386635396664353564656434356136323232336531343065653038343433343061633263
63323863663935643531303261623033356230643363313633633536623765663830323138626562
39623132393335363034613235643039323335363238323464343636333864393131356436386634
32616663653730353065393337356536626263643132366539663262343166633737343662653164
33323565313839363263326262386366663666333364313564633332363864316638646266353365
34353765343364653065643837386433383330646634353330323439313766646535636461643964
62366531653365363963356465383239373837363131326130643633643337376661323162373631
61303439386231663538626666353032383361383335336666343362336166653731336164623435
65656264316135386230306134383432343461363761353735643930663238646433373739333737
32346631646534643837626338343433393538306537646438646265326335376635363531636166
66313635666366313239333838393535353162613430616532636461363139653130393838366336
30663666353332663661376135393339366366623630386435613938646662653664646536363266
62363264623231656132393432666463326239336266326135623234343037623366653766656437
63383331653766373663333764653631313465383365356138663264313133353437393237346234
34306633663435356430343462373663663234316363636436383434633265333064353462653561
33396630656331393237353637366661386538396433373262643464326364656232343361636131
33356530386161303362623366626538313963396638616538646336303538393230353037313364
39383764386162646238373934623831313138313136613364343265643934373662346166333039
33303032636363386532366131303733383938333531616337616666353338613661646163626634
30383435366237613839616137636634636666646165353538653738303466663530353533626264
64626532393563626535306131643939653861343265366135333034623131343165643935396633
35613162633031643661326161356464616561343030636363656635383336636532393837333737
62653565663663303165643063383163303162333862303863613935613233343865663936363366
31663166663463383738383133326537313065636565393031343661613566316562333339356164
32633730316263656637363835653166373837633731613638646136663338613964663930663061
32653236646137366162663539663239363438323261343234393534386235653032346666393866
36616564343762306363383633623934303537666235363034643030396131653635613131373534
36396637623633303763643435383065653333633439646362303731363835353836663861336134
30613933303066336538333337623031663637323266373164323033366532323935373630333963
32353636356131313134363032326361306332306431623564626537393839366131336137626234
37643036383265393065376335643734303365646330643463323933363530343631376634643831
61333238373430303730373864323930616265643130636166336437363330656561303535343561
61303961366365303138373263366531303765376633656539323561363934663365323039343232
62646438633632643139326232663731343265346131343639343766613031636431326334353434
31613164666139386431623931323762373431666531306566366530383737386633303935366530
66313837313839313833396362303938383133316139306366393138626531663763343637353832
31653739623834376239666661326265626332346264363462626564653761633031656230363961
38363539653537343039303935373365383865393835386139343262346131306436303330373337
32666230366565383434363461643034323735613065646530303036353737363065366435623839
30396230346330363135383739346332653664356564666631663536613831393136653738636238
30626135646434643939303363366637646263386239646561613539303162306437313631663636
62383532656437616561636136326130666637333431343866343964393563613332373032363632
38663237396238326638376366613063366464646330333133353064663066303466626539626139
65626562303165373230613839623935383932393535653036356134653165383065636461316239
64373963333238393062
38326439643136633138386663633139616361326239646236323636313863653030393062623830
6565356135336231663237613130393761363064396138660a373061643766346634643961663465
33343865353530343331333734346432373962613036623962306163636431363465323337326465
6638613563343039660a623966366166653863363866663664653535306232376334666137376663
32363533643133623166353032393330326366373365626637306434386537646236363537313337
64646564366436356638663364326361626535363163663766353638666264313961346533373563
36363064376462313234666238323530633731336365333565666130623466643838623432366565
62636133636565323665653965646330383862306661343138316531656266306331343635333365
61633330613935316165396331376539353238343139613432623763356366653962323434306632
33323562396237633139343562306465323436326266346231663364356133316335656638623730
32653462393761303935393132346332626565326362393435643534623633343261633131346236
36613737623062626137383763646630373765613932663935353962623835353631656539373236
38363232376564383631616461393537383038666636633735376430363739356233653839623866
65643633613666316538316532613637653932356235643430646531373561383962383839383436
34333837643632363937333131393939343631323065393564386237386430393838353933363638
30363435636665646265363934386363633233623535363732636564333634623733363531313866
34636138393631326139313037356436353438666135323466373264366666363861613835353631
35623536336266643965633037383034393162643436613965356138623330643461336464613231
33303932373566386637356138663537623366656239323030656364663563393563643335376634
39346633396537646331663330656663613638323532363237363662323330383665353662366132
34346564396638306534623037623037643838373736646134646335383431616436393135646333
66316266626663356662666533333561666633303564663836643333333038613735616462663131
61656564383533623634633462323463343934303461663936356563316566393766343062336336
31323534343363626639333263393265343230366630386564626661373435646236333831333361
33333935636432303836386634616130363534363035343366663763323236666661366166326230
32636530353431316165613635643038616366386334663662373362323939613332396436333266
35393934326631386263616662333832346138356135393466356466353334353730363061636533
61353333333462353763396334663433623464303666623062356439393434333035353061623038
33666266346436643636336566303166326333613137386365646139303965613332366237336438
33666238623733646661376430643563326262663038363362643230366364623866353735323763
64313938343637393634306536346532366563636462396636353966353937393338656165323030
38383637373237666638636165393465613338616436386365623163633131396438303435356230
63343335393230336365393730356330343230326235613835656538653966316365346366666163
66373963363465353065316634326363373138323561373030646464353962313164663761376561
36346166663361636638326530633330626163323335663762326138373030363963353264373235
37636334313464376438333131303735326639386139666566336537323132336434623739396266
36326464346234356539653930336236663364303638386264656165393134313431386130366335
30633035343963656666636435306564323932666262623336613432363461373865383836383839
39393232623264636262333236616538623363306365356337666431626631653238633837343662
36336134633833356437303630383330656630316338363134316562313931323563373432356263
38646565613562303231653335323039623430613330616462303039636563643330383562303634
64633561633530623765636434393261303261623964656337373335643137316663353632336666
36306434643438323232643736313563353336353237386466343437646431376235643565633965
35366666376432646261653933393363356436653066316563376263653464663862633661623734
35626338633762633166653237323835306531616666343731623130613962376561613562643636
32303262383531636266323130623561643332613632313536633866643231333166373637663962
62333261653664636131623939616431303633393862336136626339666364396532633164383730
64383032333764306230333730396234343730656664303566643562323765663930326135366337
65363031623031366662383438306462616334646134636161343265633464353166333564383134
66646565343364303266306662353335386133306434333832396631343265316439326534396264
35373536633836613031646465336134383630396365663132376566373838303735636233333263
37663639306137623762616537666237626237376138343135626636616534313235353735313136
65613339303439343836343930613531303033336363616134313566356336386237356635303138
32636361386634643837366236663565316462323934633663346338643765366333386132653233
61316631636262626338346330643064313734323762396636316236653739383763323065313432
62616630346530616536343261383762356235663538373765376335623865393564353063373037
34353365646331346234376165323331646666363166393665666135343730643335626535356364
64663031653732303566366336353435386165383435323733663734353263323730353862363039
37633333396631363237356166366233643365363966393537343636363930313833313564383166
30343238373435326535643866613062336635666135313533666265386139653937336530353530
30346230643139316339636438346461353836316334616436396637353531313238336166663565
36623765393937323031336638326336333965303038346662363637653761303066333733633634
62373731333932643733306331666338366131333630663861663437353536303130653262633030
31313131626434386361356466646238386637376661373436636337333462303833643034386138
39616432363533346534653632653663663631333565376538663465316465653031646130313633
3733386365313864343862313363366136396432323238313163

View File

@ -1,19 +1,24 @@
- include: apt.yml
- include_tasks:
file: apt.yml
tags:
- apt
- include: timezone.yml
- include_tasks:
file: timezone.yml
tags:
- timezone
- include: users.yml
- include_tasks:
file: users.yml
tags:
- users
- include: hostname.yml
- include_tasks:
file: hostname.yml
tags:
- hostname
- include: ssh.yml
- include_tasks:
file: ssh.yml
tags:
- ssh

View File

@ -42,7 +42,6 @@
loop:
- docker-compose.yml.j2
- conf.env.j2
- nginx.conf.j2
- name: Copy nginx proxy file
copy:

View File

@ -16,7 +16,8 @@ CACHE_URL=redis://funkwhale_redis:6379/0
STATIC_ROOT={{ funkwhale_static_root }}
MUSIC_DIRECTORY_PATH={{ funkwhale_import_music_directory }}
FUNKWHALE_FRONTEND_PATH={{ funkwhale_frontend }}
# Dummy value for front container ; we have S3
MEDIA_ROOT=/media
DJANGO_SETTINGS_MODULE=config.settings.production
DJANGO_SECRET_KEY={{ funkwhale_secret_key }}
@ -35,3 +36,5 @@ AWS_DEFAULT_ACL=public-read
THROTTLING_ENABLED=false
EXTERNAL_REQUESTS_TIMEOUT=30
NGINX_MAX_BODY_SIZE=500M

View File

@ -11,14 +11,12 @@ volumes:
name: funkwhale_redis
db:
name: funkwhale_db
frontend:
name: funkwhale_frontend
static:
name: funkwhale_static
services:
celeryworker:
image: "funkwhale/funkwhale:{{ funkwhale_version }}"
image: "funkwhale/api:{{ funkwhale_version }}"
container_name: funkwhale_celeryworker
env_file:
- ./conf.env
@ -28,29 +26,28 @@ services:
- "{{ funkwhale_import_music_directory_host }}:{{ funkwhale_import_music_directory }}:ro"
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
command: celery -A funkwhale_api.taskapp worker -l INFO
command: celery -A funkwhale_api.taskapp worker -l INFO --concurrency=10
networks:
- db
restart: unless-stopped
celerybeat:
image: "funkwhale/funkwhale:{{ funkwhale_version }}"
image: "funkwhale/api:{{ funkwhale_version }}"
container_name: funkwhale_celerybeat
env_file: ./conf.env
command: celery -A funkwhale_api.taskapp beat --pidfile= -l INFO
command: celery -A funkwhale_api.taskapp beat -l INFO
networks:
- db
restart: unless-stopped
api:
image: "funkwhale/funkwhale:{{ funkwhale_version }}"
image: "funkwhale/api:{{ funkwhale_version }}"
container_name: funkwhale_api
env_file:
- ./conf.env
volumes:
- "{{ funkwhale_import_music_directory_host }}:{{ funkwhale_import_music_directory }}:ro"
- "static:{{ funkwhale_static_root }}"
- "frontend:{{ funkwhale_frontend }}"
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
labels:
@ -63,16 +60,13 @@ services:
- db
restart: unless-stopped
nginx:
image: nginx
container_name: funkwhale_nginx
front:
image: funkwhale/front:{{ funkwhale_version }}
container_name: funkwhale_front
env_file: ./conf.env
volumes:
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
- ./funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro
- "{{ funkwhale_import_music_directory_host }}:{{ funkwhale_import_music_directory }}:ro"
- "static:{{ funkwhale_static_root }}"
- "frontend:{{ funkwhale_frontend }}"
- "static:/usr/share/nginx/html/staticfiles:ro"
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
labels:

View File

@ -1,98 +0,0 @@
upstream funkwhale-api {
# depending on your setup, you may want to update this
server funkwhale_api:{{ funkwhale_api_port }};
}
# required for websocket support
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen {{ funkwhale_nginx_port }};
server_name {{ funkwhale_subdomain }}.{{ domain_name }};
# TLS
# Feel free to use your own configuration for SSL here or simply remove the
# lines and move the configuration to the previous server block if you
# don't want to run funkwhale behind https (this is not recommended)
# have a look here for let's encrypt configuration:
# https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx
root {{ funkwhale_frontend }};
# If you are using S3 to host your files, remember to add your S3 URL to the
# media-src and img-src headers (e.g. img-src 'self' https://<your-S3-URL> data:)
add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' https://s3.fr-par.scw.cloud data:; font-src 'self' data:; object-src 'none'; media-src 'self' https://s3.fr-par.scw.cloud data:";
add_header Referrer-Policy "strict-origin-when-cross-origin";
location / {
include /etc/nginx/funkwhale_proxy.conf;
# this is needed if you have file import via upload enabled
client_max_body_size {{ nginx_max_body_size }};
proxy_pass http://funkwhale-api/;
}
location /front/ {
add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; object-src 'none'; media-src 'self' data:";
add_header Referrer-Policy "strict-origin-when-cross-origin";
add_header Service-Worker-Allowed "/";
add_header X-Frame-Options "ALLOW";
alias /frontend/;
expires 30d;
add_header Pragma public;
add_header Cache-Control "public, must-revalidate, proxy-revalidate";
}
location /front/embed.html {
add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; object-src 'none'; media-src 'self' data:";
add_header Referrer-Policy "strict-origin-when-cross-origin";
add_header X-Frame-Options "ALLOW";
alias /frontend/embed.html;
expires 30d;
add_header Pragma public;
add_header Cache-Control "public, must-revalidate, proxy-revalidate";
}
location /federation/ {
include /etc/nginx/funkwhale_proxy.conf;
proxy_pass http://funkwhale-api/federation/;
}
# You can comment this if you do not plan to use the Subsonic API
location /rest/ {
include /etc/nginx/funkwhale_proxy.conf;
proxy_pass http://funkwhale-api/api/subsonic/rest/;
}
location /.well-known/ {
include /etc/nginx/funkwhale_proxy.conf;
proxy_pass http://funkwhale-api/.well-known/;
}
location ~ /_protected/media/(.+) {
internal;
# Needed to ensure DSub auth isn't forwarded to S3/Minio, see #932
proxy_set_header Authorization "";
proxy_pass $1;
}
location /_protected/music {
# this is an internal location that is used to serve
# audio files once correct permission / authentication
# has been checked on API side
# Set this to the same value as your MUSIC_DIRECTORY_PATH setting
internal;
alias {{ funkwhale_import_music_directory }};
}
location /staticfiles/ {
# django static files
alias {{ funkwhale_static_root }}/;
}
}

View File

@ -1,14 +1,13 @@
funkwhale_version: 1.2.10
funkwhale_version: 1.3.0
funkwhale_api_port: 5000
funkwhale_nginx_port: 80
funkwhale_static_root: /static
funkwhale_import_music_directory: /import
funkwhale_import_music_directory_host: "{{ funkwhale_folder_name }}/import"
funkwhale_folder_name: "{{ docker_files }}/funkwhale"
funkwhale_frontend: /frontend
funkwhale_subdomain: music
nginx_max_body_size: 100M
postgres_version: 13
postgres_version: 15
redis_version: 6
deemix_folder_path: /home/{{ base_user_name }}/deemix
deemix_songs_path: "{{ deemix_folder_path }}/songs"

View File

@ -0,0 +1,10 @@
min_cryptography_lib: 1.2.3
autorestic_base: /var/lib/autorestic
autorestic_config_path: "{{ autorestic_base }}/autorestic.yml"
autorestic_version: 1.7.7
autorestic_path: /usr/local/bin/autorestic
repository_path: /data
dbdumps_path: /dbdumps
# Default password to derive encryption key for repository (confidentiality)
restic_password: "{{ restic_password }}"

View File

@ -0,0 +1,78 @@
- name: Ensure necessary directories exists
file:
path: "{{ item }}"
state: directory
loop:
- "{{ repository_path }}"
- "{{ dbdumps_path }}"
- "{{ autorestic_base }}"
- name: Download and install restic
apt:
name: restic
update_cache: yes
- name: Install bzip2
apt:
name: bzip2
update_cache: yes
no_log: true
- name: Download autorestic
get_url:
url: "https://github.com/cupcakearmy/autorestic/releases/download/v{{ autorestic_version }}/autorestic_{{ autorestic_version }}_linux_amd64.bz2"
dest: /tmp/autorestic.bz2
- name: Extract and install autorestic executable
shell: "bzcat /tmp/autorestic.bz2 > {{ autorestic_path }}"
- name: Ensure autorestic has executable bit
file:
path: "{{ autorestic_path }}"
mode: '0755'
- name: Copy configuration
template:
src: "autorestic.yml"
dest: "{{ autorestic_config_path }}"
- name: Copy scripts
template:
src: "{{ item }}"
dest: "{{ autorestic_base }}"
mode: 0755
loop:
- backup_db.sh
- start_backup.sh
- name: Ensure scripts are executable
file:
path: "{{ autorestic_base }}/{{ item }}"
mode: 0755
loop:
- backup_db.sh
- start_backup.sh
- name: Generate systemd timer and service
template:
src: "{{ item }}"
dest: "/etc/systemd/system"
loop:
- autorestic.service
- autorestic.timer
# Remove when PR #197 is merged
- name: Initialize Restic Rest repository
shell: "RESTIC_PASSWORD='{{ restic_password }}' restic -r {{ repository_path }} init"
failed_when: false
# Waiting for PR #197 to be merged
- name: Check configuration file is correct and create repositories if needed
shell: "autorestic -c {{ autorestic_config_path }} check"
- name: Ensure timer is activated
systemd:
name: autorestic.timer
enabled: true
state: started
daemon_reload: true

View File

@ -0,0 +1,10 @@
[Unit]
Description=Backups yay
[Service]
Type=oneshot
ExecStart={{ autorestic_base }}/start_backup.sh
# fail if backup takes more than 1 day
TimeoutStartSec=86400
IPAccounting=yes
MemoryAccounting=yes

View File

@ -0,0 +1,9 @@
[Unit]
Description=Backups with autorestic
[Timer]
# Trigger the autorestic cron's check every 10 minutes
OnCalendar=*:0/10:0
[Install]
WantedBy=timers.target

View File

@ -0,0 +1,26 @@
version: 2
global:
forget:
keep-hourly: 24
keep-daily: 7
keep-weekly: 4
keep-monthly: 12
backends:
pica03:
type: local
path: {{ repository_path }}
key: {{ restic_password }}
locations:
funkwhale:
from:
- /var/lib/docker/volumes/funkwhale_static
- {{ dbdumps_path }}/funkwhale_postgres
to: pica03
cron: 0 3 * * *
forget: "yes"
hooks:
before:
- {{ autorestic_base }}/backup_db.sh funkwhale_postgres postgresql

View File

@ -0,0 +1,61 @@
#!/usr/bin/env bash
# usage: <script> <container-name> <database-type>
#
# exports the database of a running docker container in a dump in $BACKUP_DIR/$CONTAINER_NAME/
BACKUP_DIR={{ dbdumps_path }}
# Check container existence
CONTAINER="$1"
if ! docker ps | grep -q "$CONTAINER"
then
echo "The container $CONTAINER doesn't exist or doesn't run"
exit 1
fi
# Check database type
TYPE="$2"
COMMAND=""
case "$TYPE" in
postgresql)
POSTGRES_USER=$(docker exec "$CONTAINER" env | grep POSTGRES_USER | cut -d= -f2)
COMMAND="pg_dumpall -c -U $POSTGRES_USER"
EXTENSION=sql
;;
mariadb)
MARIADB_USER=$(docker exec "$CONTAINER" env | grep MYSQL_USER | cut -d= -f2)
MARIADB_PASSWORD=$(docker exec "$CONTAINER" env | grep MYSQL_PASSWORD | cut -d= -f2)
COMMAND="mysqldump -u $MARIADB_USER --password=$MARIADB_PASSWORD --all-databases"
EXTENSION=sql
;;
mongodb)
COMMAND="mongodump --archive"
EXTENSION=mongodump
;;
ldap-config)
COMMAND="slapcat -n 0"
EXTENSION=config.ldif
;;
ldap-content)
COMMAND="slapcat -n 1"
EXTENSION=content.ldif
;;
*)
echo "I don't know $TYPE database type."
exit 1
esac
# Ensure directory exists
mkdir -p "$BACKUP_DIR/$CONTAINER"
# Export database
docker exec "$CONTAINER" $COMMAND > "$BACKUP_DIR/$CONTAINER/dump.$EXTENSION"
exit $?

View File

@ -0,0 +1,25 @@
#!/usr/bin/env sh
if [ ! -f /tmp/last_autorestic_check_date ]
then
touch /tmp/last_autorestic_check_date
fi
current_date=$(date +"%D")
last_autorestic_check_date=$(cat /tmp/last_autorestic_check_date)
{{ autorestic_path }} -c {{ autorestic_config_path }} --ci exec -av -- unlock
#Check only one time a day
if [ "$current_date" != "$last_autorestic_check_date" ]
then
#todo: use exec -- check when PR #253 is merged (more verbose)
{{ autorestic_path }} -c {{ autorestic_config_path }} check
if [ $? -ne 0 ]
then
exit
fi
echo $current_date > /tmp/last_autorestic_check_date
fi
{{ autorestic_path }} -vvv -c {{ autorestic_config_path }} --ci cron