From fa6f3ba4b1da35d11ea3bbda9411336b6898c786 Mon Sep 17 00:00:00 2001 From: Quentin Duchemin Date: Tue, 27 Apr 2021 03:16:49 +0200 Subject: [PATCH] Add Funkwhale with S3 storage --- all.yml | 4 +- inv/host_vars/new.chosto.me/secrets.yml | 62 ++++++----- inv/host_vars/new.chosto.me/vars.yml | 2 +- roles/funkwhale/files/funkwhale_proxy.conf | 19 ++++ roles/funkwhale/tasks/main.yml | 50 +++++++++ roles/funkwhale/templates/conf.env.j2 | 29 +++++ .../funkwhale/templates/docker-compose.yml.j2 | 102 ++++++++++++++++++ roles/funkwhale/templates/nginx.conf.j2 | 98 +++++++++++++++++ roles/funkwhale/vars/main.yml | 12 +++ 9 files changed, 347 insertions(+), 31 deletions(-) create mode 100644 roles/funkwhale/files/funkwhale_proxy.conf create mode 100644 roles/funkwhale/tasks/main.yml create mode 100644 roles/funkwhale/templates/conf.env.j2 create mode 100644 roles/funkwhale/templates/docker-compose.yml.j2 create mode 100644 roles/funkwhale/templates/nginx.conf.j2 create mode 100644 roles/funkwhale/vars/main.yml diff --git a/all.yml b/all.yml index e2bb27d..b7c8da0 100644 --- a/all.yml +++ b/all.yml @@ -25,4 +25,6 @@ - role: "gitea" tags: ["docker", "gitea"] - role: "nextcloud" - tags: ["nextcloud", "gitea"] + tags: ["nextcloud", "docker"] + - role: "funkwhale" + tags: ["funkwhale", "docker"] diff --git a/inv/host_vars/new.chosto.me/secrets.yml b/inv/host_vars/new.chosto.me/secrets.yml index 8425b8a..bef4171 100644 --- a/inv/host_vars/new.chosto.me/secrets.yml +++ b/inv/host_vars/new.chosto.me/secrets.yml @@ -1,30 +1,34 @@ $ANSIBLE_VAULT;1.1;AES256 -32623335343331343131646165313031333361363864396334303961373133633337376638326363 -3135306436633631386361623766626239663839343831340a346566633339666133353765313838 -65643931626663643233306330336133373335326536376664323263336336396633316431393963 -3263306663383437320a643062616435326531663161393938313639383364386331383134376234 -66343365663032666434653861346265376131313864363665386633306263663937386331663633 -34613566646133316631323961363631643666643163356361393933353661643934626266353633 -35656430323363633961323230373230663664376339373965353761623163383934356532356633 -65633765613436353766386666363139353838663963643764623232363432636363373765376363 -35313930626366613661663263356232303933616131313666326339616633383165656136653434 -61616564643638366534326238333134623234333562636161343730396531663131636166303663 -36323333383531393435666431386431353638346437633130363730656433663031346362653339 -66623066383234623364653861633231316137353636623761346132373937653638376139643132 -66656263383131333233373631666462663266343462336634643532633638306462323166303161 -33643962363939313131666133393637636161393436336436613162376332386636653361393638 -65313932363433616564646138393465316137663937613037656663376330373136323032643032 -62626534376633393034613839336632666164363864366332366439313664343630336339633464 -35333232353061366463616232353430356131656633353866643062653531306162653463393039 -66663435343333383834306234316634656166333432306238306162613836303365643338633635 -34323235316231356230336466303032623235303930653439323236363533376437613363636163 -38666163383062303034653662343530333737316263666163656437343439663962353134313530 -38333034316432383330643966353030623637326334663330333530633534643662663835346465 -36663131363136396230653963393037313032333863353866663864643065383765306664333564 -38303861383636636434303038633835643934373439313438653761323165366430633165396537 -37633638393730653865313238393239393739366237353633366164393235393435313361343834 -34633233353037393138353664376330643965303666313138346333343935323531346631316563 -37306535643031623837396233366665306665626532356532386561356534653738333439646165 -65663131643162643061313266326638616531643063303361666464616166303263353561356462 -61343338343137323463353539643136313334646364313437663764653037633937373838613634 -3135 +34363462333030653462383364323934653331333861333732303365626439666666393232376139 +6161356563623135646365323133326333383734383136340a643335623334363066353930303638 +38653862376330353361613661383330343338633963333538623934396537356137643833663262 +3431653035643063330a383634633966643133386236303064663666303935333636386532363363 +33646631343761363133646635663836313832616264313134616635373230393935396330373936 +36656666623631636230356665366532613230396565336136316530633432326665366135376238 +34633666623063383632663333366137666265363663323264643631323463633865336635636435 +35616631623532303536613064353135353034333739656432393835303839333165633135663934 +31663233656137653230343036666336386361393937383636336536396539303131393133653234 +30343030373863636232643635656664643561383264643465363163656131323731326361623639 +31663362363337306238616564336330303462346537393336363266323031653166323366333466 +36376433373663666535623864303533353837663064623432306363356638363634323831663437 +31663462323666633835663831653439306438376662343762663136613532366136636661383166 +33303563613436323334366532316336346635356433663766363831646336336665653365616663 +32303165313935326462393833363563313235386637353761306262353733316265383133303037 +35373338653931383463323533646262653066323164313939336336376262353066363339653938 +62383035653333663663336364646634336563366131653665373033333365386562333966353063 +36383964633561326262616439383739343736343362363264393137366662306630656364333532 +63346331636266626637666264343263303534313038386263666634353330643938393236336361 +34356661343334316162313030636533643064383531653836356366623432383066333033663536 +32656639323030653635636265343731336531646539356261383139663261386439376237396536 +62666130353038386635333265376630376165376433336436636331316531663935663339356436 +35303765303031323564333232363335643235376366613931653035313035663737353937393737 +66353663643735623762303234663762356136326133656338656664313637346136376266383636 +36386637326430626264666362643639636533373530366337373561643335363236646237636338 +62393531643663646433303233366233366536373865613331383539616238303135383665343930 +33303930633533333637343634393038356235646533613766623436306666306166383632303233 +38343063636236663432333336393838373637633737363865373261343965623736326433313937 +34323037326362323032356232373065666639616362393536653663316439376662636431626238 +32353838666535633831353538306634636562343633656663343131386462656536633663333235 +38386435313336613962313665616132323431356333353861386663313562373837663966623532 +65363438643666326163393761626231386331343435636562336363643733353439326230326637 +61633531316335396662663539366264633034373333336638323734336364323038 diff --git a/inv/host_vars/new.chosto.me/vars.yml b/inv/host_vars/new.chosto.me/vars.yml index ed44830..b3ec301 100644 --- a/inv/host_vars/new.chosto.me/vars.yml +++ b/inv/host_vars/new.chosto.me/vars.yml @@ -14,6 +14,6 @@ compose_version: "3.7" traefik_network: proxy -domain_name: new.chosto.me +domain_name: chosto.me letsencrypt_email: quentinduchemin@tuta.io diff --git a/roles/funkwhale/files/funkwhale_proxy.conf b/roles/funkwhale/files/funkwhale_proxy.conf new file mode 100644 index 0000000..dccc782 --- /dev/null +++ b/roles/funkwhale/files/funkwhale_proxy.conf @@ -0,0 +1,19 @@ +# use this one if you put the nginx container behind another proxy +# you will have to set some headers on this proxy as well to ensure +# everything works correctly, you can use the ones from the funkwhale_proxy.conf file +# at https://dev.funkwhale.audio/funkwhale/funkwhale/blob/develop/deploy/funkwhale_proxy.conf +# your proxy will also need to support websockets + +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; + +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; +proxy_set_header X-Forwarded-Host $http_x_forwarded_host; +proxy_set_header X-Forwarded-Port $http_x_forwarded_port; +proxy_redirect off; + +# websocket support +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $connection_upgrade; diff --git a/roles/funkwhale/tasks/main.yml b/roles/funkwhale/tasks/main.yml new file mode 100644 index 0000000..d11de75 --- /dev/null +++ b/roles/funkwhale/tasks/main.yml @@ -0,0 +1,50 @@ +--- +- name: Create Funkwhale directory + file: + path: "{{ funkwhale_folder_name }}" + state: directory + owner: "{{ base_user_name }}" + group: "{{ base_user_name }}" + mode: 0755 + +- name: Copy Traefik templates (nginx conf and Compose) + template: + src: "{{ item }}" + # Remove .j2 extension + dest: "{{ funkwhale_folder_name }}/{{ (item | splitext)[0] }}" + owner: "{{ base_user_name }}" + group: "{{ base_user_name }}" + mode: 0644 + loop: + - docker-compose.yml.j2 + - conf.env.j2 + - nginx.conf.j2 + +- name: Copy nginx proxy file + copy: + src: funkwhale_proxy.conf + dest: "{{ funkwhale_folder_name }}/funkwhale_proxy.conf" + owner: "{{ base_user_name }}" + group: "{{ base_user_name }}" + mode: 0644 + +- name: Start Funkwhale database + community.docker.docker_compose: + project_src: "{{ funkwhale_folder_name }}" + services: + - db + pull: yes + state: present + +- name: Run migrations + shell: + cmd: docker-compose run --rm api python manage.py migrate + chdir: "{{ funkwhale_folder_name }}" + +- name: Run all funkwhale containers + community.docker.docker_compose: + project_src: "{{ funkwhale_folder_name }}" + remove_orphans: yes + pull: yes + recreate: smart + state: present diff --git a/roles/funkwhale/templates/conf.env.j2 b/roles/funkwhale/templates/conf.env.j2 new file mode 100644 index 0000000..3e670d5 --- /dev/null +++ b/roles/funkwhale/templates/conf.env.j2 @@ -0,0 +1,29 @@ +FUNKWHALE_API_IP=127.0.0.1 +FUNKWHALE_API_PORT={{ funkwhale_api_port }} +FUNKWHALE_WEB_WORKERS=4 +FUNKWHALE_HOSTNAME={{ funkwhale_subdomain }}.{{ domain_name }} +FUNKWHALE_PROTOCOL=https + +EMAIL_CONFIG=smtp+tls://{{ funkwhale_subdomain }}@{{ domain_name }}:mD32H&Y2X$9XPFQtS!tq@mail.gandi.net:587 +DEFAULT_FROM_EMAIL={{ funkwhale_subdomain }}@{{ domain_name }} + +DATABASE_URL=postgresql://funkwhale:{{ funkwhale_db_password }}@funkwhale_postgres:5432/funkwhale + +REVERSE_PROXY_TYPE=nginx + +CACHE_URL=redis://funkwhale_redis:6379/0 + +STATIC_ROOT={{ funkwhale_static_root }} +MUSIC_DIRECTORY_PATH={{ funkwhale_import_music_directory }} +FUNKWHALE_FRONTEND_PATH={{ funkwhale_frontend }} + +DJANGO_SETTINGS_MODULE=config.settings.production +DJANGO_SECRET_KEY={{ funkwhale_secret_key }} + +NGINX_MAX_BODY_SIZE={{ nginx_max_body_size}} + +AWS_ACCESS_KEY_ID={{ scaleway_s3_id }} +AWS_SECRET_ACCESS_KEY={{ scaleway_s3_key }} +AWS_STORAGE_BUCKET_NAME=celiglyphe +AWS_S3_ENDPOINT_URL=https://s3.fr-par.scw.cloud +PROXY_MEDIA=false diff --git a/roles/funkwhale/templates/docker-compose.yml.j2 b/roles/funkwhale/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..14092ba --- /dev/null +++ b/roles/funkwhale/templates/docker-compose.yml.j2 @@ -0,0 +1,102 @@ +version: "{{ compose_version }}" + +networks: + proxy: + name: "{{ traefik_network }}" + db: + name: funkwhale_db + +volumes: + redis: + name: funkwhale_redis + db: + name: funkwhale_db + frontend: + name: funkwhale_frontend + static: + name: funkwhale_static + +services: + celeryworker: + image: "funkwhale/funkwhale:{{ funkwhale_version }}" + container_name: funkwhale_celeryworker + env_file: + - ./conf.env + environment: + - C_FORCE_ROOT=true + volumes: + - "{{ funkwhale_import_music_directory_host }}:{{ funkwhale_import_music_directory }}:ro" + command: celery -A funkwhale_api.taskapp worker -l INFO + networks: + - db + restart: unless-stopped + + celerybeat: + image: "funkwhale/funkwhale:{{ funkwhale_version }}" + container_name: funkwhale_celerybeat + env_file: ./conf.env + command: celery -A funkwhale_api.taskapp beat --pidfile= -l INFO + networks: + - db + restart: unless-stopped + + api: + image: "funkwhale/funkwhale:{{ funkwhale_version }}" + container_name: funkwhale_api + env_file: + - ./conf.env + volumes: + - "{{ funkwhale_import_music_directory_host }}:{{ funkwhale_import_music_directory }}:ro" + - "static:{{ funkwhale_static_root }}" + - "frontend:{{ funkwhale_frontend }}" + labels: + traefik.http.routers.funkwhale_api.entrypoints: websecure + traefik.http.routers.funkwhale_api.rule: "Host(`api.{{ funkwhale_subdomain }}.{{ domain_name }}`)" + traefik.http.services.funkwhale_api.loadbalancer.server.port: "{{ funkwhale_api_port }}" + traefik.enable: true + networks: + - proxy + - db + restart: unless-stopped + + nginx: + image: nginx + container_name: funkwhale_nginx + env_file: ./conf.env + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + - ./funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro + - "{{ funkwhale_import_music_directory_host }}:{{ funkwhale_import_music_directory }}:ro" + - "static:{{ funkwhale_static_root }}" + - "frontend:{{ funkwhale_frontend }}" + labels: + traefik.http.routers.funkwhale.entrypoints: websecure + traefik.http.routers.funkwhale.rule: "Host(`{{ funkwhale_subdomain }}.{{ domain_name }}`)" + traefik.http.services.funkwhale.loadbalancer.server.port: "{{ funkwhale_nginx_port }}" + traefik.enable: true + networks: + - proxy + restart: unless-stopped + + redis: + image: "redis:{{ redis_version }}" + container_name: funkwhale_redis + env_file: ./conf.env + volumes: + - redis:/data + networks: + - db + restart: unless-stopped + + db: + image: "postgres:{{ postgres_version }}" + container_name: funkwhale_postgres + environment: + POSTGRES_USER: funkwhale + POSTGRES_DB: funkwhale + POSTGRES_PASSWORD: "{{ funkwhale_db_password }}" + volumes: + - db:/var/lib/postgresql/data + networks: + - db + restart: unless-stopped diff --git a/roles/funkwhale/templates/nginx.conf.j2 b/roles/funkwhale/templates/nginx.conf.j2 new file mode 100644 index 0000000..6dcbbab --- /dev/null +++ b/roles/funkwhale/templates/nginx.conf.j2 @@ -0,0 +1,98 @@ +upstream funkwhale-api { + # depending on your setup, you may want to update this + server funkwhale_api:{{ funkwhale_api_port }}; +} + + +# required for websocket support +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen {{ funkwhale_nginx_port }}; + server_name {{ funkwhale_subdomain }}.{{ domain_name }}; + + # TLS + # Feel free to use your own configuration for SSL here or simply remove the + # lines and move the configuration to the previous server block if you + # don't want to run funkwhale behind https (this is not recommended) + # have a look here for let's encrypt configuration: + # https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx + + root {{ funkwhale_frontend }}; + + # If you are using S3 to host your files, remember to add your S3 URL to the + # media-src and img-src headers (e.g. img-src 'self' https:// data:) + + add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' https://s3.fr-par.scw.cloud data:; font-src 'self' data:; object-src 'none'; media-src 'self' https://s3.fr-par.scw.cloud data:"; + add_header Referrer-Policy "strict-origin-when-cross-origin"; + + + location / { + include /etc/nginx/funkwhale_proxy.conf; + # this is needed if you have file import via upload enabled + client_max_body_size {{ nginx_max_body_size }}; + proxy_pass http://funkwhale-api/; + } + + location /front/ { + add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; object-src 'none'; media-src 'self' data:"; + add_header Referrer-Policy "strict-origin-when-cross-origin"; + add_header Service-Worker-Allowed "/"; + add_header X-Frame-Options "ALLOW"; + alias /frontend/; + expires 30d; + add_header Pragma public; + add_header Cache-Control "public, must-revalidate, proxy-revalidate"; + } + + location /front/embed.html { + add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; object-src 'none'; media-src 'self' data:"; + add_header Referrer-Policy "strict-origin-when-cross-origin"; + + add_header X-Frame-Options "ALLOW"; + alias /frontend/embed.html; + expires 30d; + add_header Pragma public; + add_header Cache-Control "public, must-revalidate, proxy-revalidate"; + } + + location /federation/ { + include /etc/nginx/funkwhale_proxy.conf; + proxy_pass http://funkwhale-api/federation/; + } + + # You can comment this if you do not plan to use the Subsonic API + location /rest/ { + include /etc/nginx/funkwhale_proxy.conf; + proxy_pass http://funkwhale-api/api/subsonic/rest/; + } + + location /.well-known/ { + include /etc/nginx/funkwhale_proxy.conf; + proxy_pass http://funkwhale-api/.well-known/; + } + + location ~ /_protected/media/(.+) { + internal; + # Needed to ensure DSub auth isn't forwarded to S3/Minio, see #932 + proxy_set_header Authorization ""; + proxy_pass $1; + } + + location /_protected/music { + # this is an internal location that is used to serve + # audio files once correct permission / authentication + # has been checked on API side + # Set this to the same value as your MUSIC_DIRECTORY_PATH setting + internal; + alias {{ funkwhale_import_music_directory }}; + } + + location /staticfiles/ { + # django static files + alias {{ funkwhale_static_root }}}/; + } +} diff --git a/roles/funkwhale/vars/main.yml b/roles/funkwhale/vars/main.yml new file mode 100644 index 0000000..9b94955 --- /dev/null +++ b/roles/funkwhale/vars/main.yml @@ -0,0 +1,12 @@ +funkwhale_version: 1.1.1 +funkwhale_api_port: 5000 +funkwhale_nginx_port: 80 +funkwhale_static_root: /static +funkwhale_import_music_directory: /import +funkwhale_import_music_directory_host: "{{ funkwhale_folder_name }}/import" +funkwhale_folder_name: "{{ docker_files }}/funkwhale" +funkwhale_frontend: /frontend +funkwhale_subdomain: music +nginx_max_body_size: 100M +postgres_version: 13 +redis_version: 6