Merge branch 'develop' into obf-new

.github/labeler.yml

@@ -1,6 +1,10 @@
-# Add 'label1' to any changes within 'example' folder or any subfolders
-github_actions:
-- .github/**/*
+# Add labels to any any pull request with changes to the specified paths
+
+# Pull requests that update GitHub Actions code. If you navigate to the folder, you will have a README of what it does
+GitHub Actions:
+- changed-files:
+  - any-glob-to-any-file: '.github/**/*'
 
 postmortems:
-- docs/reports/**/*
+- changed-files:
+  - any-glob-to-any-file: 'docs/reports/**/*'

.github/workflows/generate-docs.yml

@@ -32,7 +32,7 @@ jobs:
         # we only deploy on push to main
         if: |
           github.event_name == 'push' && github.event.ref == 'refs/heads/develop'
-        uses: JamesIves/github-pages-deploy-action@v4.5.0
+        uses: JamesIves/github-pages-deploy-action@v4.6.1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           branch: gh-pages # The branch the action should deploy to.

.github/workflows/label.yml

@@ -17,6 +17,6 @@ jobs:
       contents: read
       pull-requests: write
     steps:
-    - uses: actions/labeler@v4
+    - uses: actions/labeler@v5
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/readme-writer.yml

@@ -14,7 +14,7 @@ jobs:
         # we use a personal access token of openfoodfacts-bot to allow bypassing branch protection rules
         token: "${{ secrets.README_WRITER_PAT }}"
     - name: Set up python
-      uses: actions/setup-python@v4.3.0
+      uses: actions/setup-python@v5
       with:
         python-version: '3.9'
     - run: |

README.md

@@ -1,7 +1,8 @@
 # OpenFoodFacts Infrastructure
 
 Sysadmin repository for the various parts of the Open Food Facts infrastructure.
-We have a [specific repository regarding monitoring](https://github.com/openfoodfacts/openfoodfacts-monitoring)
+
+We also have a [specific repository regarding monitoring](https://github.com/openfoodfacts/openfoodfacts-monitoring)
 
 ## Current priorities
 
@@ -36,8 +37,10 @@ Link to [Github Page](https://openfoodfacts.github.io/openfoodfacts-infrastructu
 
 The infrastructure documentation is as follows:
 
-- [Free Datacenter](./docs/free-datacenter.md) - Data center with main production servers
+- [Overview](./docs/overview.md)
 
+- [Mail](./docs/mail.md) - servers mail setup
+- [Free Datacenter](./docs/free-datacenter.md) - Data center with main production servers
 - [Linux Server](./docs/linux-server.md) - servers general setup
 - [Mail](./docs/mail.md) - servers mail setup
 - [An introduction to ZFS](./docs/zfs-overview.md) - ZFS is much used in our infrastructure
@@ -79,6 +82,7 @@ Also look at all install and post-mortem reports in [docs/reports](./docs/report
 
 ### Virtual Machines
 
+<!-- This table is auto-generated by the readme-writer action  -->
 <!-- VM table -->
 |                                                                      Title                                                                      |State |              OS              |      CPU #      |                              RAM                              |                                                    SSD (Local)                                                    |    HDD (Remote)     |                                       Services                                        |
 |-------------------------------------------------------------------------------------------------------------------------------------------------|------|------------------------------|-----------------|---------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------------|---------------------------------------------------------------------------------------|

confs/matomo/matomo/archive-2.env

@@ -0,0 +1,5 @@
+# ENV file called by [email protected]
+# This is for website archiving of openfoodfacts.org
+# Note: comma is mandatory for --force-idsites to be taken into account correctly
+MATOMO_OPTIONS="--force-idsites=2, --concurrent-archivers=0"
+

confs/matomo/matomo/archive-5.env

@@ -0,0 +1,5 @@
+# ENV file called by [email protected]
+# This is for generating archive of mobile app
+# Note: comma is mandatory for --force-idsites to be taken into account correctly
+MATOMO_OPTIONS="--force-idsites=5, --concurrent-archivers=0"
+

confs/matomo/matomo/archive-main.env

@@ -0,0 +1,4 @@
+# ENV file called by [email protected]
+# This is for main archive generation, that is excluding specific high traffic services
+MATOMO_OPTIONS="--skip-idsites=5,2 --concurrent-archivers=0"
+

confs/matomo/mysql/mariadb.conf.d/90-off-configs.cnf

@@ -1,10 +1,12 @@
 [mysqld]
 # following https://matomo.org/faq/troubleshooting/faq_183/
 wait_timeout=28800
+interactive_timeout=28800
 net_read_timeout=28800
 net_write_timeout=28800
 innodb_log_file_size=2048M
 max_allowed_packet=2048M
+max_connections=400
 
 # following https://matomo.org/faq/on-premise/how-to-configure-matomo-for-speed/
 # see also https://mariadb.com/resources/blog/10-database-tuning-tips-for-peak-workloads/

confs/matomo/systemd/mariadb.service.d/override.conf

@@ -1,3 +1,3 @@
 [Unit]
-OnFailure=email-failures@mariadb-%H.service
+[email protected]
 

confs/matomo/systemd/matomo-archive.service → confs/matomo/systemd/matomo-archive@.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Archive matomo requests
+Description=Archive matomo requests (for %i parameters)
 Requires=mariadb.service
 
 # email on failure
@@ -12,10 +12,11 @@ Environment=TZ=UTC
 Type=oneshot
 User=www-data
 Group=www-data
-ExecStart=/usr/bin/php /var/www/html/matomo/console core:archive --url=http://analytics.openfoodfacts.org/ -n --ignore-warn
-StandardOutput=append:/var/log/matomo/matomo-archive.log
-StandardError=append:/var/log/matomo/matomo-archive-err.log
+# defines MATOMO_OPTIONS thanks to specific environment
+EnvironmentFile=/etc/matomo/archive-%i.env
+ExecStart=/usr/bin/php /var/www/html/matomo/console core:archive --url=http://analytics.openfoodfacts.org/ -n --ignore-warn ${MATOMO_OPTIONS}
+StandardOutput=append:/var/log/matomo/matomo-archive-%i.log
+StandardError=append:/var/log/matomo/matomo-archive-%i-err.log
 # place a timeout for bad cases as this process sometimes hangs
 # note: we have to use TimeoutStartSec as it's a Type=oneshot service
-TimeoutStartSec=5h
-
+TimeoutStartSec=5h40m

confs/matomo/systemd/matomo-archive@.timer

@@ -0,0 +1,13 @@
+[Unit]
+Description=Run Matomo archive process every 6h
+
+[Timer]
+# every 6h
+OnBootSec=5m
+# relaunch every 6h after activating
+# this is correlated to a 5h40 timeout in the service
+OnUnitActiveSec=6h
+
+[Install]
+WantedBy=timers.target
+

confs/matomo/systemd/php7.3-fpm.service.d/override.conf

@@ -1,3 +1,3 @@
 [Unit]
-[email protected]-%H.service
+[email protected]
 

scripts/ovh3/nginx-conf/static-off → confs/off1-reverse-proxy/nginx/sites-available/images-off

@@ -1,33 +1,56 @@
-# définition d'une liste d'upstream (priorité au début)
 log_format combined_upstream '$remote_addr - $remote_user [$time_local] '
                     '"$request" $status $body_bytes_sent '
                     '"$http_referer" "$http_user_agent" c=$upstream_cache_status u=$upstream_addr t=$request_time';
 
-upstream openfoodfacts {
-	server 10.0.0.3:443 weight=100;
-	server off1.openfoodfacts.org:443;
-
-	keepalive 16;
+# mapping to expected image size (approximately)
+# for smart rate limiting
+map $uri $response_size {
+	default small_size;
+	# full image either have full, or ar 1.jpg
+	"~*\.full\.jpg" big_size;
+	"~*/\d+\.jpg" big_size;
+}
+map $response_size $big_rate_key {
+	default $binary_remote_addr;
+	small_size '';  # skipped
+}
+map $response_size $small_rate_key {
+	default $binary_remote_addr;
+	big_size '';  # skipped
 }
 
+# rate limit, differentiate between small an full req
+# small (loading search page: 100 product images + burst for assets) * 60
+limit_req_zone $small_rate_key zone=small_size:10m rate=6000r/m;
+# some full image per minutes should be enough
+limit_req_zone $big_rate_key zone=big_size:10m rate=100r/m;
+
+# DRY RUN until we decide to activate it
+limit_req_dry_run on;
+
+# use a clear status
+limit_req_status 429;
+
 proxy_cache_path
-	/dev/shm/off-static
-	keys_zone=off-static:10m
+	/dev/shm/off-images
+	keys_zone=off-images:10m
 	levels=1:2
 	inactive=24h
 	max_size=4G;
 
-# https://static.openfoodfacts.org
+# https://images.openfoodfacts.org
+
 server {
         listen 443 ssl http2;
         listen [::]:443 ssl http2;
-        server_name static.openfoodfacts.org images.openfoodfacts.org;
+        server_name images.openfoodfacts.org;
 
-        access_log /rpool/logs-nginx/static-access.log combined_upstream buffer=256K flush=1s;
+        access_log /var/log/nginx/images.openfoodfacts.access.log combined_upstream buffer=256K flush=1s;
+        error_log /var/log/nginx/images.openfoodfacts.error.log;
 
         ssl_certificate /etc/letsencrypt/live/images.openfoodfacts.org/fullchain.pem; # managed by Certbot
         ssl_certificate_key /etc/letsencrypt/live/images.openfoodfacts.org/privkey.pem; # managed by Certbot
-	ssl_trusted_certificate /etc/nginx/acme.sh/live/openfoodfacts.org/ca.pem;
+	ssl_trusted_certificate /etc/letsencrypt/live/images.openfoodfacts.org/chain.pem;
 
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
@@ -40,16 +63,19 @@ server {
 	resolver 9.9.9.9 8.8.8.8 valid=300s;
 	resolver_timeout 5s;
 
-	root	/rpool/off/ ;
-
-
-	if ($http_referer ~* (jobothoniel.com) ) {
-	     return 403;
-	}
+	root	/mnt/off/ ;
 
 	location / {
-		# test en local, puis sur off1
-		try_files		$uri @off1;
+		# rate limit based on size
+		# NGINX actually tracks requests at millisecond granularity
+		# so eg 6000r/m means 100r/s which means 1r each 10 ms,
+		# but requests tends to come simultaneously
+		# The simple way to avoid caring about this is to have burst=rate
+		limit_req zone=big_size burst=100 nodelay;
+		limit_req zone=small_size burst=6000 nodelay;
+
+		# test en local, puis sur l'original
+		try_files		$uri @proxy2;
 		sendfile		on;
 		sendfile_max_chunk	1m;
 		tcp_nopush		on;
@@ -61,13 +87,13 @@ server {
 	        add_header Access-Control-Expose-Headers 'Content-Length,Content-Range';
 	}
 
-	location @off1 {
-		proxy_pass		https://off1.openfoodfacts.org;
+	location @proxy2 {
+		proxy_pass		http://10.1.0.113:80;
 #		proxy_next_upstream	error	 http_404;
 		proxy_http_version	1.1;
 		proxy_set_header	Connection	"";
 
-		proxy_cache		off-static;
+		proxy_cache		off-images;
 		proxy_cache_valid	200  1d;
 		proxy_cache_key		$request_uri;
 		add_header		X-Cache-Status	$upstream_cache_status;

confs/off1/sanoid/syncoid-args.conf

@@ -8,7 +8,7 @@
 # obf
 --no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/obf zfs-hdd/obf
 # opf
---no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/opf zfs-hdd/obf
+--no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/opf zfs-hdd/opf
 # opff
 --no-sync-snap --no-privilege-elevation --recursive [email protected]:zfs-hdd/opff zfs-hdd/opff
 # off2 rpool backups

confs/off2-reverse-proxy/systemd/system/nginx.service.d/override.conf

@@ -1,2 +1,7 @@
+[Service]
+# let nginx refine this
+LimitNOFILE=65535
+
 [Unit]
 [email protected]
+Restart=always

confs/off2-reverse-proxy/systemd/system/prometheus-nginx-exporter.service.d/override.conf

@@ -0,0 +1,3 @@
+[Service]
+# ONLY listen on 127.0.0.1 for security reasons
+Environment="LISTEN_ADDRESS=127.0.0.1:9113"

confs/off2/nginx/sites-available/static-off

@@ -34,7 +34,7 @@ limit_req_status 429;
 
 upstream openfoodfacts {
 	server 10.0.0.2:443 weight=100;
-	server off1.openfoodfacts.org:443;
+	server proxy2.openfoodfacts.org:443;
 
 	keepalive 16;
 }
@@ -86,8 +86,8 @@ server {
 		limit_req zone=big_size burst=100 nodelay;
 		limit_req zone=small_size burst=6000 nodelay;
 
-		# test en local, puis sur off1
-		try_files		$uri @off1;
+		# test en local, puis sur l'original
+		try_files		$uri @proxy2;
 		sendfile		on;
 		sendfile_max_chunk	1m;
 		tcp_nopush		on;
@@ -99,8 +99,8 @@ server {
 	        add_header Access-Control-Expose-Headers 'Content-Length,Content-Range';
 	}
 
-	location @off1 {
-		proxy_pass		https://off1.openfoodfacts.org;
+	location @proxy2 {
+		proxy_pass		https://proxy2.openfoodfacts.org;
 #		proxy_next_upstream	error	 http_404;
 		proxy_http_version	1.1;
 		proxy_set_header	Connection	"";

confs/ovh3/sanoid/sanoid.conf

@@ -48,8 +48,8 @@
 
 
 [rpool/off-backups]
-  # for the  parent dataset only, 
-  # must be before recursive declaration 
+  # for the  parent dataset only,
+  # must be before recursive declaration
   use_template=local_sys
   recursive=no
 
@@ -59,7 +59,7 @@
   recursive=yes
 
 
-# Template to regularly snapshot 
+# Template to regularly snapshot
 [template_local_data]
   # How often snapshots should be taken under an hour
   frequent_period=30

confs/proxy-off/nginx/free-exporters.openfoodfacts.org

@@ -19,9 +19,16 @@ server {
 
 # map from service to exporter
 map $uri $exporter {
+    # mongodb
     "/mongo/mongodb/metrics" 10.1.0.102:9216;
+    # nginx on this proxy
+    "/proxy/nginx/metrics" 127.0.0.1:9113;
+    # nginx on off
+    "/off/nginx/metrics" 10.1.0.113:9113;
 }
 
+#log_format debug_exporter "$time_local $scheme://$host:$server_port$request_uri $exporter";
+
 # https server
 server {
     listen 443 ssl http2;
@@ -33,15 +40,17 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/free-exporters.openfoodfacts.org/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/free-exporters.openfoodfacts.org/chain.pem;
 
-    # logs location
-    access_log  /var/log/nginx/free-exporters.log  main;
+    # logs location, avoid access_log, no need
+    #access_log  /var/log/nginx/free-exporters.log  debug_exporter;
+    #access_log  /var/log/nginx/free-exporters.log  main;
+    access_log  off;
     error_log   /var/log/nginx/free-exporters.errors.log;
 
     # protect
     auth_basic "Exporter metrics";
     auth_basic_user_file "/etc/nginx/.htpasswd/free-exporters";
 
-    # mongodb exporter
+    # url to join is defined thanks to exporter variable, itself based upon uri
     location / {
 
         if ($exporter = "") {

confs/proxy-off/nginx/log_format.conf

@@ -1,3 +1,3 @@
 log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                   '$status $body_bytes_sent "$http_referer" '
-                  '"$http_user_agent" "$http_x_forwarded_for"';
+                  '"$http_user_agent" "$http_x_forwarded_for" $upstream_cache_status [$upstream_response_time]';