makefile: add rund and some other log things

Makefile

@@ -10,6 +10,7 @@ MODULES += mobilizon
 MODULES += gitea
 MODULES += nitter
 MODULES += pixelfed
+MODULES += bookwyrm
 
 include env.production
 domain_name := $(DOMAIN_NAME)
@@ -27,6 +28,10 @@ DOCKER = \
 
 run:
 	$(DOCKER) up
+rund:
+	$(DOCKER) up -d
+stop:
+	$(DOCKER) stop
 down:
 	$(DOCKER) down
 nginx-shell:
@@ -34,7 +39,7 @@ nginx-shell:
 nginx-logs:
 	$(DOCKER) logs -f --tail 1000 nginx
 mastodon-logs:
-	$(DOCKER) logs -f --tail 1000 mastodon
+	$(DOCKER) logs -f --tail 1000 mastodon mastodon-sidekiq
 grafana-shell:
 	$(DOCKER) exec grafana bash
 hedgedoc-shell:
@@ -47,6 +52,8 @@ keycloak-rebuild:
 mastodon-es-rebuild:
 	$(DOCKER) create mastodon-es
 	$(DOCKER) restart mastodon-es
+mastodon-restart:
+	$(DOCKER) restart mastodon
 mastodon-shell:
 	$(DOCKER) exec mastodon bash
 mastodon-streaming-shell:
@@ -57,6 +64,8 @@ nextcloud-shell:
 	$(DOCKER) exec nextcloud bash
 matrix-logs:
 	$(DOCKER) logs --tail 100 -f matrix-synapse
+mastodon-sidekiq-logs:
+	$(DOCKER) logs --tail 100 -f mastodon-sidekiq
 nextcloud-logs:
 	$(DOCKER) logs -f nextcloud
 nginx-build: secrets/nginx

bookwyrm.yaml

@@ -0,0 +1,142 @@
+version: '3'
+
+services:
+  bookwyrm-db:
+    container_name: bookwyrm-db
+    image: postgres
+    env_file: bookwyrm/env
+    volumes:
+      - ./data/bookwyrm/pgdata:/var/lib/postgresql/data
+
+  bookwyrm-redis_activity:
+    container_name: bookwyrm-redis_activity
+    image: redis
+    env_file: bookwyrm/env
+    command: redis-server --requirepass redispassword123 --appendonly yes
+    volumes:
+      - ./bookwyrm/redis.conf:/etc/redis/redis.conf:ro
+      - ./data/bookwyrm/redis_activity_data:/data
+    restart: on-failure
+
+  bookwyrm-redis_broker:
+    container_name: bookwyrm-redis_broker
+    image: redis
+    command: redis-server --requirepass redispassword123 --appendonly yes
+    env_file: bookwyrm/env
+    volumes:
+      - ./bookwyrm/redis.conf:/etc/redis/redis.conf:ro
+      - ./data/bookwyrm/redis_broker_data:/data
+    restart: on-failure
+
+  bookwyrm-web:
+    container_name: bookwyrm-web
+    image: osresearch/bookwyrm:oidc
+    command: python manage.py runserver 0.0.0.0:8000
+    env_file: bookwyrm/env
+    volumes:
+      - ./data/bookwyrm/static_volume:/app/static
+      - ./data/bookwyrm/media_volume:/app/images
+    environment:
+      - DB_INIT=True
+      - DOMAIN=${BOOKWYRM_HOSTNAME}.${DOMAIN_NAME}
+      - EMAIL=books@${DOMAIN_NAME}
+      - EMAIL_HOST=${SMTP_SERVER}
+      - EMAIL_PORT=${SMTP_PORT}
+      - EMAIL_HOST_USER=${SMTP_USER}
+      - EMAIL_HOST_PASSWORD=${SMTP_PASSWORD}
+      - EMAIL_USE_TLS=true
+      - EMAIL_USE_SSL=false
+      - EMAIL_SENDER_NAME=books
+      - EMAIL_SENDER_DOMAIN=${DOMAIN_NAME}
+      - OIDC_ENABLED=true
+      - OIDC_CLIENT_ID=bookwyrm
+      - OIDC_CLIENT_SECRET=${BOOKWYRM_CLIENT_SECRET}
+      - OIDC_OP_BASE_URL=${KEYCLOAK_BASE_URL}
+    depends_on:
+      - bookwyrm-db
+      - bookwyrm-celery_worker
+      - bookwyrm-redis_activity
+
+  bookwyrm-celery_worker:
+    container_name: bookwyrm-worker
+    image: osresearch/bookwyrm:oidc
+    command: celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority
+    env_file: bookwyrm/env
+    volumes:
+      - ./data/bookwyrm/static_volume:/app/static
+      - ./data/bookwyrm/media_volume:/app/images
+    environment:
+      - DOMAIN=${BOOKWYRM_HOSTNAME}.${DOMAIN_NAME}
+      - EMAIL=books@${DOMAIN_NAME}
+      - EMAIL_HOST=${SMTP_SERVER}
+      - EMAIL_PORT=${SMTP_PORT}
+      - EMAIL_HOST_USER=${SMTP_USER}
+      - EMAIL_HOST_PASSWORD=${SMTP_PASSWORD}
+      - EMAIL_USE_TLS=true
+      - EMAIL_USE_SSL=false
+      - EMAIL_SENDER_NAME=books
+      - EMAIL_SENDER_DOMAIN=${DOMAIN_NAME}
+    depends_on:
+      - bookwyrm-db
+      - bookwyrm-redis_broker
+    restart: on-failure
+
+  bookwyrm-celery_beat:
+    container_name: bookwyrm-beat
+    image: osresearch/bookwyrm:oidc
+    command: celery -A celerywyrm beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler
+    env_file: bookwyrm/env
+    volumes:
+      - ./data/bookwyrm/static_volume:/app/static
+      - ./data/bookwyrm/media_volume:/app/images
+    environment:
+      - DOMAIN=${BOOKWYRM_HOSTNAME}.${DOMAIN_NAME}
+      - EMAIL=books@${DOMAIN_NAME}
+      - EMAIL_HOST=${SMTP_SERVER}
+      - EMAIL_PORT=${SMTP_PORT}
+      - EMAIL_HOST_USER=${SMTP_USER}
+      - EMAIL_HOST_PASSWORD=${SMTP_PASSWORD}
+      - EMAIL_USE_TLS=true
+      - EMAIL_USE_SSL=false
+      - EMAIL_SENDER_NAME=books
+      - EMAIL_SENDER_DOMAIN=${DOMAIN_NAME}
+    depends_on:
+      - bookwyrm-celery_worker
+    restart: on-failure
+
+  bookwyrm-flower:
+    container_name: bookwyrm-flower
+    image: osresearch/bookwyrm:oidc
+    command: celery -A celerywyrm flower --basic_auth=admin:${BOOKWYRM_ADMIN_PASSWORD}
+    env_file: bookwyrm/env
+#    ports:
+#      - ${FLOWER_PORT}:${FLOWER_PORT}
+    volumes:
+      - ./data/bookwyrm/static_volume:/app/static
+      - ./data/bookwyrm/media_volume:/app/images
+    environment:
+      - DOMAIN=${BOOKWYRM_HOSTNAME}.${DOMAIN_NAME}
+      - EMAIL=books@${DOMAIN_NAME}
+      - EMAIL_HOST=${SMTP_SERVER}
+      - EMAIL_PORT=${SMTP_PORT}
+      - EMAIL_HOST_USER=${SMTP_USER}
+      - EMAIL_HOST_PASSWORD=${SMTP_PASSWORD}
+      - EMAIL_USE_TLS=true
+      - EMAIL_USE_SSL=false
+      - EMAIL_SENDER_NAME=books
+      - EMAIL_SENDER_DOMAIN=${DOMAIN_NAME}
+    depends_on:
+      - bookwyrm-db
+      - bookwyrm-redis_broker
+    restart: on-failure
+
+  nginx:
+    volumes:
+      - ./bookwyrm/nginx.conf:/etc/nginx/templates/bookwyrm.conf.template:ro
+      - ./data/bookwyrm/static_volume:/bookwyrm/app/static:ro
+      - ./data/bookwyrm/media_volume:/bookwyrm/app/images:ro
+
+  # add the subdomain client secrets to the keycloak-setup volume
+  keycloak:
+    volumes:
+      - ./bookwyrm/keycloak.sh:/keycloak-setup/bookwyrm.sh:ro

bookwyrm/env

@@ -0,0 +1,102 @@
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG=false
+USE_HTTPS=true
+
+# Instance defualt language (see options at bookwyrm/settings.py "LANGUAGES"
+LANGUAGE_CODE="en-us"
+# Used for deciding which editions to prefer
+DEFAULT_LANGUAGE="English"
+
+## Leave unset to allow all hosts
+# ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
+
+MEDIA_ROOT=images/
+
+# Database configuration
+PGPORT=5432
+POSTGRES_PASSWORD=securedbypassword123
+POSTGRES_USER=bookwyrm
+POSTGRES_DB=bookwyrm
+POSTGRES_HOST=bookwyrm-db
+
+# Redis activity stream manager
+MAX_STREAM_LENGTH=200
+REDIS_ACTIVITY_HOST=bookwyrm-redis_activity
+REDIS_ACTIVITY_PASSWORD=redispassword123
+# Optional, use a different redis database (defaults to 0)
+# REDIS_ACTIVITY_DB_INDEX=0
+
+# Redis as celery broker
+REDIS_BROKER_HOST=bookwyrm-redis_broker
+REDIS_BROKER_PASSWORD=redispassword123
+# Optional, use a different redis database (defaults to 0)
+# REDIS_BROKER_DB_INDEX=0
+
+# Monitoring for celery
+FLOWER_PORT=8888
+FLOWER_USER=admin
+
+
+# Query timeouts
+SEARCH_TIMEOUT=5
+QUERY_TIMEOUT=5
+
+# Thumbnails Generation
+ENABLE_THUMBNAIL_GENERATION=false
+
+# S3 configuration
+USE_S3=false
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+# Commented are example values if you use a non-AWS, S3-compatible service
+# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
+# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
+
+
+# Preview image generation can be computing and storage intensive
+ENABLE_PREVIEW_IMAGES=false
+
+# Specify RGB tuple or RGB hex strings,
+# or use_dominant_color_light / use_dominant_color_dark
+PREVIEW_BG_COLOR=use_dominant_color_light
+# Change to #FFF if you use use_dominant_color_dark
+PREVIEW_TEXT_COLOR=#363636
+PREVIEW_IMG_WIDTH=1200
+PREVIEW_IMG_HEIGHT=630
+PREVIEW_DEFAULT_COVER_COLOR=#002549
+
+# Below are example keys if you want to enable automatically
+# sending telemetry to an OTLP-compatible service. Many of
+# the main monitoring apps have OLTP collectors, including
+# NewRelic, DataDog, and Honeycomb.io - consult their
+# documentation for setup instructions, and what exactly to
+# put below!
+#
+# Service name is an arbitrary tag that is attached to any
+# data sent, used to distinguish different sources. Useful
+# for sending prod and dev metrics to the same place and
+# keeping them separate, for instance!
+
+# API endpoint for your provider
+OTEL_EXPORTER_OTLP_ENDPOINT=
+# Any headers required, usually authentication info
+OTEL_EXPORTER_OTLP_HEADERS=
+# Service name to identify your app
+OTEL_SERVICE_NAME=
+
+# Set HTTP_X_FORWARDED_PROTO ONLY to true if you know what you are doing.
+# Only use it if your proxy is "swallowing" if the original request was made
+# via https. Please refer to the Django-Documentation and assess the risks
+# for your instance:
+# https://docs.djangoproject.com/en/3.2/ref/settings/#secure-proxy-ssl-header
+HTTP_X_FORWARDED_PROTO=false

bookwyrm/keycloak.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -x
+
+client_id=$(client-create bookwyrm "$BOOKWYRM_HOSTNAME.$DOMAIN_NAME" "$BOOKWYRM_CLIENT_SECRET" </dev/null)
+
+echo '{"name":"admin"}' | kcadm.sh create -r "$REALM" "clients/$client_id/roles" -f -
+echo '{"name":"moderator"}' | kcadm.sh create -r "$REALM" "clients/$client_id/roles" -f -
+echo '{"name":"editor"}' | kcadm.sh create -r "$REALM" "clients/$client_id/roles" -f -

bookwyrm/nginx.conf

@@ -0,0 +1,79 @@
+limit_req_zone $binary_remote_addr zone=loginlimit:10m rate=1r/s;
+
+server {
+    server_name ${BOOKWYRM_HOSTNAME} ${BOOKWYRM_HOSTNAME}.${DOMAIN_NAME};
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    #include /etc/nginx/mime.types;
+    #default_type application/octet-stream;
+
+    gzip on;
+    gzip_disable "msie6";
+
+    proxy_read_timeout 1800s;
+    chunked_transfer_encoding on;
+
+    client_body_buffer_size     10M;
+    client_max_body_size        10M;
+
+    set $skip_cache 0;
+    if ($request_method = POST) {
+        set $skip_cache 1;
+    }
+
+    if ($http_cookie ~* "session") {
+        set $skip_cache 1;
+    }
+
+
+    location ~ ^/(login[^-/]|password-reset|resend-link|2fa-check) {
+        limit_req zone=loginlimit;
+
+        proxy_pass http://bookwyrm-web:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location ~ ^/(api|oidc|preferences) {
+        proxy_pass http://bookwyrm-web:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+        access_log off;
+    }
+
+    location / {
+        proxy_cache mycache;
+        add_header X-Cache-Status $upstream_cache_status;
+        proxy_ignore_headers Cache-Control Set-Cookie;
+        #proxy_ignore_headers Cache-Control;
+
+	# logged in sessions and other reasons to bypass the cache
+	proxy_no_cache      $skip_cache;
+	proxy_cache_bypass  $skip_cache;
+  
+        proxy_cache_valid any 1m;
+
+        proxy_pass http://bookwyrm-web:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location /images/ {
+        alias /bookwyrm/app/images/;
+        access_log off;
+    }
+
+    location /static/ {
+        alias /bookwyrm/app/static/;
+        access_log off;
+    }
+
+    include /etc/nginx/includes/ssl.conf;
+}

bookwyrm/redis.conf

@@ -0,0 +1,9 @@
+bind 127.0.0.1 ::1
+protected-mode yes
+port 6379
+
+rename-command FLUSHDB ""
+rename-command FLUSHALL ""
+rename-command DEBUG ""
+rename-command CONFIG ""
+rename-command SHUTDOWN ""

env.production

@@ -21,7 +21,9 @@ MOBILIZON_HOSTNAME=events
 PIXELFED_HOSTNAME=pixelfed
 PROMETHEUS_HOSTNAME=metrics
 NITTER_HOSTNAME=nitter
+BOOKWYRM_HOSTNAME=books
 
+KEYCLOAK_BASE_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect
 AUTH_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/auth
 TOKEN_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/token
 USERINFO_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/userinfo

gitea.yaml

@@ -71,5 +71,4 @@ services:
   # add the gitea client secrets to the keycloak-setup volume
   keycloak:
     volumes:
-      - ./data/gitea/secrets:/run/secrets/gitea:ro
       - ./gitea/keycloak.sh:/keycloak-setup/gitea.sh:ro

grafana.yaml

@@ -39,4 +39,3 @@ services:
   keycloak:
     volumes:
       - ./grafana/keycloak.sh:/keycloak-setup/grafana.sh:ro
-      - ./data/grafana/secrets:/run/secrets/grafana:ro

hedgedoc.yaml

@@ -54,4 +54,3 @@ services:
   keycloak:
     volumes:
       - ./hedgedoc/keycloak.sh:/keycloak-setup/hedgedoc.sh:ro
-      - ./data/hedgedoc/secrets:/run/secrets/hedgedoc:ro

keycloak.yaml

@@ -45,7 +45,7 @@ services:
         - ./keycloak/entrypoint-setup.sh:/setup.sh:ro
         - ./keycloak/mail-setup.sh:/keycloak-setup/mail-setup.sh:ro
         - ./keycloak/mapper-setup.sh:/keycloak-setup/mapper-setup.sh:ro
-        - ./data/keycloak/secrets:/run/secrets/keycloak-secrets:ro
+        - ./secrets:/run/secrets:ro
       depends_on:
         - keycloak-db
 

mastodon.yaml

@@ -203,7 +203,6 @@ services:
   keycloak:
     volumes:
       - ./mastodon/keycloak.sh:/keycloak-setup/mastodon.sh:ro
-      - ./data/mastodon/secrets:/run/secrets/mastodon:ro
 
 #networks:
 #  external_network:

matrix.yaml

@@ -52,4 +52,3 @@ services:
   keycloak:
     volumes:
       - ./matrix/keycloak.sh:/keycloak-setup/matrix.sh:ro
-      - ./data/matrix/secrets:/run/secrets/matrix:ro

mobilizon.yaml

@@ -59,4 +59,3 @@ services:
   keycloak:
     volumes:
       - ./mobilizon/keycloak.sh:/keycloak-setup/mobilizon.sh:ro
-      - ./data/mobilizon/secrets:/run/secrets/mobilizon:ro

nextcloud.yaml

@@ -44,4 +44,3 @@ services:
   keycloak:
     volumes:
       - ./nextcloud/keycloak.sh:/keycloak-setup/nextcloud.sh:ro
-      - ./data/nextcloud/secrets:/run/secrets/nextcloud:ro

pixelfed.yaml

@@ -87,4 +87,3 @@ services:
   keycloak:
     volumes:
       - ./pixelfed/keycloak.sh:/keycloak-setup/pixelfed.sh:ro
-      - ./secrets/pixelfed:/run/secrets/pixelfed:ro

secrets/README

@@ -0,0 +1 @@
+# Do not check in anything in this directory