update 2024-04-22

Changes:
- add jinja2 template engine instead of string-replacing
- fix certbot cron usage
- replace json servers configuration with yaml

.env.example

@@ -1 +1 @@
-EMAIL=your@email.com
+EMAIL=your@email.com

.gitignore

@@ -1,3 +1,4 @@
 /nginx/domains.txt
 /nginx/servers.json
-*.env
+/nginx/servers.yaml
+*.env

certbot/Dockerfile

@@ -6,15 +6,11 @@ ARG EMAIL
 
 RUN apk add certbot bash
 
-RUN echo "#!/bin/sh" > /usr/bin/update_certificates && \
+RUN mkdir -p /etc/letsencrypt /etc/letsencrypt.bak /ssl/.well-known && \
-    echo "certbot renew --quiet" >> /usr/bin/update_certificates && \
-    echo "cp -rL /etc/letsencrypt/live/* /ssl/" >> /usr/bin/update_certificates && \
     \
-    mkdir -p /etc/letsencrypt /ssl/.well-known && \
+    echo "webroot-path = /ssl/" > /etc/letsencrypt.bak/cli.ini && \
     \
-    echo "webroot-path = /ssl/" > /etc/letsencrypt/cli.ini && \
+    echo '15 2 */7 * * /run_once' > /etc/crontabs/root && \
-    \
-    echo '15 2 */7 * * /usr/bin/update_certificates' > /etc/crontabs/root && \
     \
     echo "echo 'running with cron'" > /run_with_cron && \
     echo "cp /etc/letsencrypt.bak/cli.ini /etc/letsencrypt/cli.ini" >> /run_with_cron && \
@@ -25,7 +21,7 @@ RUN echo "#!/bin/sh" > /usr/bin/update_certificates && \
     echo "if [ ! -f /ssl/domains.txt ]; then echo 'No domains.txt file found in /ssl, exiting' && exit 1; fi" >> /run_once && \
     echo 'for domain in $(cat /ssl/domains.txt); do case $domain in "#"*) :; ;; *) certbot certonly -n --authenticator webroot -d $domain; ;; esac; done' >> /run_once && \
     echo "cp -rL /etc/letsencrypt/live/* /ssl/" >> /run_once && \
-    chmod +x /usr/bin/update_certificates
+    chmod +x /run_once
 
 RUN certbot register --email $EMAIL --non-interactive --agree-tos
 

certbot_manual.sh

@@ -8,7 +8,7 @@ if [ "$EMAIL" = '' ]; then
 fi
 
 docker build --tag certbot_manual_test --build-arg EMAIL="$EMAIL" certbot
-echo "ececute 'cat /run_once' to get commands list"
+echo "execute 'cat /run_once' to get commands list"
 echo "for wildcard domains use manual mode with dns challange: 'certbot certonly -d '*.domain' --manual --preferred-challenges dns"
 docker run -it --rm -v $SSL_VOLUME_NAME:/ssl -v $LETSENCRYPT_VOLUME_NAME:/etc/letsencrypt --entrypoint /bin/bash certbot_manual_test
-docker rmi certbot_manual_test
+docker rmi certbot_manual_test

docker-compose.yml

@@ -1,40 +1,38 @@
-version: '3.4'
+name: ssl_nginx
-
+
-name: ssl_nginx
+services:
-
+  certbot:
-services:
+    container_name: 'global-certbot'
-  certbot:
+    build:
-    container_name: 'global-certbot'
+      context: ./certbot
-    build:
+      args:
-      context: ./certbot
+        EMAIL: ${EMAIL}
-      args:
+    volumes:
-        EMAIL: ${EMAIL}
+      - ssl:/ssl
-    volumes:
+      - letsencrypt:/etc/letsencrypt
-      - ssl:/ssl
+    depends_on:
-      - letsencrypt:/etc/letsencrypt
+      - nginx
-    depends_on:
+    restart: unless-stopped
-      - nginx
+
-    restart: unless-stopped
+  nginx:
-
+    container_name: 'global-nginx'
-  nginx:
+    build: ./nginx
-    container_name: 'global-nginx'
+    volumes:
-    build: ./nginx
+      - ssl:/ssl
-    volumes:
+      - letsencrypt:/etc/letsencrypt
-      - ssl:/ssl
+    ports:
-      - letsencrypt:/etc/letsencrypt
+      - 80:80
-    ports:
+      - 443:443
-      - 80:80
+    networks:
-      - 443:443
+      - hosting_net
-    networks:
+    restart: unless-stopped
-      - hosting_net
+
-    restart: unless-stopped
+volumes:
-
+  ssl:
-volumes:
+    name: ssl_nginx_ssl
-  ssl:
+  letsencrypt:
-    name: ssl_nginx_ssl
+    name: ssl_nginx_letsencrypt 
-  letsencrypt:
+
-    name: ssl_nginx_letsencrypt
+networks:
-
+  hosting_net:
-networks:
+    external: true
-  hosting_net:
-    external: true

nginx/Dockerfile

@@ -1,22 +1,22 @@
-FROM python:3.10-alpine as builder
+FROM python:3.11-alpine as builder
+
+RUN pip3 install --no-cache-dir pyyaml jinja2
 
 COPY add_servers.py /add_servers.py
-COPY servers.json /servers.json
+COPY servers.yaml /servers.yaml
 COPY domains.txt /domains.txt
-COPY nginx.conf /nginx.conf
+COPY nginx.conf.j2 /nginx.conf.j2
 
-RUN python /add_servers.py --nginx /nginx.conf --domains_list_txt /domains.txt --servers_config_json servers.json --certificates_path /ssl
+RUN python /add_servers.py --nginx-template /nginx.conf.j2 -o /nginx.conf --domains_list_txt /domains.txt --servers-config servers.yaml --certificates-path /ssl
 
+# service
 
 FROM nginx:alpine
 
-COPY proxy_common.conf /etc/nginx/proxy_common.conf
-COPY server_common.conf /etc/nginx/server_common.conf
-
 COPY --from=builder /nginx.conf /etc/nginx/nginx.conf
 COPY domains.txt /domains.txt
 
-RUN echo "16 2 */7 * * nginx -s reload" > /etc/crontabs/certbot && \
+RUN echo "16 2 */7 * * nginx -s reload" > /etc/crontabs/root && \
     \
     echo "cp /domains.txt /ssl/domains.txt" > /entrypoint && \
     echo "crond" >> /entrypoint && \

nginx/add_servers.py

@@ -1,121 +1,87 @@
 """Add domain servers to nginx.conf executable script."""
+from __future__ import annotations
+
 import argparse
+import os
 from dataclasses import dataclass, field
-import json
+from typing import Any
-import sys
 
-REDIRECT_TEMPLATE = "\n".join(
+import jinja2
-    (
+import yaml
-        "\tlocation / {{",
-        "\t\tresolver 127.0.0.11 valid=30s;",
-        "\t\tset $host_{i} {redirection_host};",
-        "\t\tproxy_pass $host_{i}/;",
-        "\t\tinclude proxy_common.conf;",
-        "\t}}",
-    )
-)
-
-SERVER_HTTPS_TEMPLATE = "\n".join(
-    (
-        "server {{",
-        "\tinclude server_common.conf;",
-        "\tlisten              443 ssl;",
-        "\t",
-        "\tserver_name         {server_name};",
-        "\tssl_certificate     {certificates_path}/{certificate}/fullchain.pem;",
-        "\tssl_certificate_key {certificates_path}/{certificate}/privkey.pem;",
-        "\t",
-        "{options}",
-        "{redirection}",
-        "}}",
-    )
-)
-
-SERVER_HTTP_TEMPLATE = "\n".join(
-    (
-        "server {{",
-        "\tinclude server_common.conf;",
-        "\t",
-        "\tserver_name         {server_name};",
-        "{options}",
-        "{redirection}",
-        "}}",
-    )
-)
 
 
 @dataclass
 class Server:
     """Server entry for nginx.conf file."""
 
-    server_name: str
+    name: str
+    all_names: str | None = None
     redirect: str | None = None
-    certificate: str | None = None
+    certificate_dir: str | None = None
-    options: list[str] = field(default_factory=list)
+    port: int = None
+    ssl_port: int = None
+    server_options: list[str] = field(default_factory=list)
+    location_options: list[str] = field(default_factory=list)
 
     certificates_path: str = "/etc/letsencrypt/live"
 
     def __post_init__(self) -> None:
-        if self.options is None:
+        if self.server_options is None:
-            self.options = []
+            self.server_options = []
+        if self.location_options is None:
+            self.location_options = []
+        if self.all_names is None:
+            self.all_names = self.name
+        if self.port is None:
+            self.port = 80
+        if self.ssl_port is None:
+            self.ssl_port = 443
 
-    def format(self, i: int, indent: str = "    ", base_indent: int = 1) -> str:
+    def to_dict(self) -> dict:
-        """Format server to place inside nginx.conf"""
+        """Convert server class to dict for nginx.conf.j2 jinja2-template"""
-        res = (indent * base_indent) + (
+        return {
-            SERVER_HTTPS_TEMPLATE if self.certificate is not None else SERVER_HTTP_TEMPLATE
+            "name": self.name,
-        ).replace("\n", "\n" + indent * base_indent).replace("\t", indent).format(
+            "all_names": self.all_names,
-            server_name=self.server_name,
+            "redirect": self.redirect,
-            certificate=self.certificate,
+            "server_options": self.server_options,
-            redirection=(
+            "location_options": self.location_options,
-                REDIRECT_TEMPLATE.replace("\n", "\n" + indent * base_indent)
+            "certificate_dir": self.certificate_dir,
-                .replace("\t", indent)
+            "port": self.port,
-                .format(i=i, redirection_host=self.redirect)
+            "ssl_port": self.ssl_port,
-                if self.redirect is not None
+        }
-                else ""
-            ),
-            options=(
-                ("\n" + indent * (base_indent + 1))
-                + ("\n" + indent * (base_indent + 1)).join(
-                    ("\n" + indent * (base_indent + 1)).join(option.split("\n")) for option in self.options
-                )
-            ),
-            certificates_path=self.certificates_path,
-        )
-        res = "\n".join(line for line in res.split("\n") if line.strip() != "")
-        return res
 
 
 @dataclass
 class CLIParams:
     """add_servers CLI parameters"""
 
-    nginx: str
+    nginx_template: str
     domains_list_txt: str
-    servers_config_json: str
+    servers_config: str
     certificates_path: str
     http_only: bool
-    dry_run: bool
+    output: str | None
 
 
 def main() -> None:
     """Parse arguments and add domains to nginx.conf"""
     parser = argparse.ArgumentParser("add-servers", description="Add domain servers to a given nginx.conf file")
-    parser.add_argument("--nginx", "-f", required=True, help="Path to nginx.conf file to edit inplace")
+    parser.add_argument("--nginx-template", "-f", required=True, help="Path to nginx.conf.j2 template file")
     parser.add_argument(
         "--domains_list_txt",
         "-d",
         required=True,
-        help="Path to domains list with ssl certificates ",
+        help="Path to file with domains which have ssl certificates",
     )
     parser.add_argument(
-        "--servers_config_json",
+        "--servers-config",
         "-s",
         required=False,
         default=None,
-        help='Path to domains json {"domain": {"redirect": "redirection_path", "options": []}};',
+        help="Path to servers configuration yaml file",
     )
     parser.add_argument(
-        "--certificates_path",
+        "--certificates-path",
         "-c",
         required=False,
         default="/etc/letsencrypt/live",
@@ -126,85 +92,77 @@ def main() -> None:
         action="store_true",
         help="Remove certificates usage from servers section",
     )
-    parser.add_argument(
+    parser.add_argument("--output", "-o", help="Path to nginx.conf output file")
-        "--dry-run",
-        action="store_true",
-        help="Print servers section and quit withot making changes",
-    )
 
     args: CLIParams = parser.parse_args()
 
-    replacement_substring = "# <place_for_servers>"
-
-    with open(args.nginx, "r", encoding="utf-8") as file:
-        nginx_config = file.read()
-    if nginx_config.count(replacement_substring) != 1:
-        print(
-            f"Error. File must contain exactly one '{replacement_substring}' substring to replace with servers. Exiting"
-        )
-        sys.exit(1)
-
     if args.domains_list_txt is not None:
         with open(args.domains_list_txt, "r", encoding="utf-8") as file:
-            domains_certs_list = [
+            domains_with_certs = [
                 domain.strip() for domain in file.readlines() if domain.strip() != "" and not domain.startswith("#")
             ]
     else:
-        domains_certs_list = []
+        domains_with_certs = []
 
     nginx_servers: list[Server] = []
-    if args.servers_config_json is not None:
+    if args.servers_config is not None:
-        with open(args.servers_config_json, "r", encoding="utf-8") as file:
+        with open(args.servers_config, "r", encoding="utf-8") as file:
-            for server_name, params in json.load(file).items():
+            data: dict = yaml.safe_load(file)
-                server_name: str
+            resolver: str = data.get("resolver", "127.0.0.1")
-                params: dict[str, str]
+            acme_challenge_location: str | None = data.get("acme_challenge_location")
-                nginx_servers.append(
+            servers: dict[str, dict[str, Any]] = data["servers"]
-                    Server(
+
-                        (server_name if "*" not in server_name else f"{server_name} {server_name.replace('*.', '')}"),
+        for server_name, params in servers.items():
-                        params.get("redirect"),
+            nginx_servers.append(
-                        (
+                Server(
-                            None
+                    name=(server_name if "*" not in server_name else f"{server_name} {server_name.replace('*.', '')}"),
-                            if args.http_only
+                    all_names=params.get("all_names"),
-                            else server_name.replace("*.", "")
+                    redirect=params.get("redirect"),
-                            if server_name in domains_certs_list
+                    certificate_dir=_get_certificate_path(
-                            else server_name[server_name.find(".") + 1 :]
+                        args.http_only, domains_with_certs, args.certificates_path, server_name
-                            if "." in server_name
+                    ),
-                            and f"*.{server_name[server_name.find('.') + 1:]}" in domains_certs_list
+                    port=params.get("port"),
-                            else None
+                    ssl_port=params.get("ssl_port"),
-                        ),
+                    server_options=params.get("server_options"),
-                        params.get("options"),
+                    location_options=params.get("location_options"),
-                        args.certificates_path,
-                    )
                 )
-    for domain in domains_certs_list:
+            )
+    for domain in domains_with_certs:
         if not any(
             (
-                domain == server.server_name
+                domain == server.name
-                or (
+                or (domain == f"*.{server.name[server.name.find('.') + 1:]}" if "." in server.name else False)
-                    domain == f"*.{server.server_name[server.server_name.find('.') + 1:]}"
-                    if "." in server.server_name
-                    else False
-                )
             )
             for server in nginx_servers
         ):
-            nginx_servers.append(
+            nginx_servers.append(Server(name=domain))
-                Server(domain, None, domain if not args.http_only else None, certificates_path=args.certificates_path)
-            )
 
-    nginx_servers_part = "\n\n".join(server.format(i) for i, server in enumerate(nginx_servers))
+    with open(args.nginx_template, "r", encoding="utf-8") as file:
+        template = jinja2.Environment().from_string(file.read())
 
-    print(f"Using following servers part for nginx.conf:\n\n{nginx_servers_part}")
+    result = template.render(
+        resolver=resolver,
+        acme_challenge_location=acme_challenge_location,
+        servers=[server.to_dict() for server in nginx_servers],
+    )
 
-    if not args.dry_run:
+    print(result)
-        config_backup_filename = f"{args.nginx}.bak"
-        print(f"Backing up old nginx config {args.nginx} as {config_backup_filename}")
 
-        with open(config_backup_filename, "w", encoding="utf-8") as file:
+    if args.output is not None:
-            file.write(nginx_config)
+        with open(args.output, "w", encoding="utf-8") as file:
+            file.write(result)
 
-        with open(args.nginx, "w", encoding="utf-8") as file:
+
-            file.write(nginx_config.replace(replacement_substring, nginx_servers_part.lstrip()))
+def _get_certificate_path(
+    http_only: bool, domains_with_certs: list[str], base_certs_path: str, server_name: str
+) -> str | None:
+    if http_only:
+        return None
+    if server_name in domains_with_certs:
+        return os.path.join(base_certs_path, server_name.replace("*.", ""))
+    if "." in server_name and f"*.{server_name[server_name.find('.') + 1:]}" in domains_with_certs:
+        return os.path.join(base_certs_path, server_name[server_name.find(".") + 1 :])
+    return None
 
 
 if __name__ == "__main__":

nginx/domains.txt.example

@@ -1,3 +1,5 @@
 your.domain.to_redirect
-your.other_domain
+your_other.doma.in
-#commented.domain
+*.doma.in
+#commented.domain
+domain.without.redirect

nginx/get-certificates.Dockerfile

@@ -6,6 +6,8 @@ COPY nginx.conf /nginx.conf
 
 RUN python /add_servers.py --nginx /nginx.conf --domains_list_txt /domains.txt --http-only
 
+# service
+
 FROM nginx:alpine
 
 COPY proxy_common.conf /etc/nginx/proxy_common.conf
@@ -14,7 +16,7 @@ COPY server_common.conf /etc/nginx/server_common.conf
 COPY --from=builder /nginx.conf /etc/nginx/nginx.conf
 COPY domains.txt /domains.txt
 
-RUN echo "(sleep 60 && killall nginx) &" > /entrypoint && \
+RUN echo "(sleep 120 && killall nginx) &" > /entrypoint && \
     echo "cp /domains.txt /ssl/domains.txt" >> /entrypoint && \
     echo "nginx -g 'daemon off;'" >> /entrypoint && \
     echo "nginx" >> /entrypoint

nginx/nginx.conf

@@ -1,27 +0,0 @@
-user  nginx;
-worker_processes  auto;
-
-error_log  /var/log/nginx/error.log notice;
-pid        /var/run/nginx.pid;
-
-events {
-    worker_connections  1024;
-}
-
-http {
-    ssl_session_cache   shared:SSL:10m;
-    ssl_session_timeout 10m;
-    server_tokens off;
-    gzip on;
-    proxy_connect_timeout 300;
-    proxy_send_timeout 600;
-    proxy_read_timeout 600;
-    send_timeout 600;
-    client_max_body_size 1100M;
-
-    server {
-        return 404;
-    }
-
-    # <place_for_servers>
-}

nginx/nginx.conf.j2

@@ -0,0 +1,82 @@
+{#- variables ~ examples: #}
+{#-  acme_challenge_location ~ /ssl/: #}
+{#-  resolver ~ 127.0.0.11: #}
+{#-  servers ~ ["name": ..., ("redirect": ..., "server_options": ..., "location_options": ..., "all_names": ..., "port": ..., "ssl_port": ...)]: #}
+{#- #}user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    ssl_session_cache   shared:SSL:10m;
+    ssl_session_timeout 10m;
+    server_tokens off;
+    gzip on;
+
+    proxy_connect_timeout 300;
+    proxy_send_timeout 600;
+    proxy_read_timeout 600;
+    send_timeout 600;
+    client_max_body_size 500M;
+
+    server {
+        return 404;
+    }
+
+    {%- for server in servers %}
+    server {
+        listen              {{ server["port"] or 80 }};
+        {%- if server["certificate_dir"] is not none %}
+        listen              {{ server["ssl_port"] or 443 }} ssl;
+        {%- endif %}
+        keepalive_timeout   70;
+
+        server_name         {{ server["all_names"] or server["name"] }};
+
+        {%- if acme_challenge_location is defined %}
+        {# #}
+        location /.well-known/acme-challenge {
+            root {{ acme_challenge_location }};
+        }
+        {%- endif %}
+
+        {%- if server["server_options"]|length > 0 %}
+        {# #}
+        {%- for server_option in server["server_options"] %}
+        {{ server_option }}
+        {%- endfor %}
+        {%- endif %}
+
+        {%- if server["certificate_dir"] is not none %}
+        {# #}
+        ssl_certificate     {{ server["certificate_dir"] }}/fullchain.pem;
+        ssl_certificate_key {{ server["certificate_dir"] }}/privkey.pem;
+        {%- endif %}
+
+        {%- if server["redirect"] is not none %}
+        {# #}
+        location / {
+            resolver {{ resolver }};
+            set $host_{{ loop.index }} {{ server["redirect"] }};
+            proxy_pass $host_{{ loop.index }};
+
+            proxy_set_header Host $host:$server_port;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Real-IP $remote_addr;
+
+            {%- if server["location_options"]|length > 0 %}
+            {# #}
+            {%- for location_option in server["location_options"] %}
+            {{ location_option }}
+            {%- endfor %}
+            {%- endif %}
+        }
+        {%- endif %}
+    }
+    {%- endfor %}
+}

nginx/proxy_common.conf

@@ -1,3 +0,0 @@
-                proxy_set_header Host $host:$server_port;
-                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-                proxy_set_header X-Real-IP $remote_addr;

nginx/server_common.conf

@@ -1,6 +0,0 @@
-        listen              80;
-        keepalive_timeout   70;
-
-        location /.well-known/acme-challenge {
-            root /ssl/;
-        }

nginx/servers.json.example

@@ -1,8 +0,0 @@
-{
-    "server.name": {
-        "redirect": "http://local.address",
-        "options": [
-            "other_server_level_nginx_options here;"
-        ]
-    }
-}

nginx/servers.yaml.example

@@ -0,0 +1,23 @@
+resolver: 127.0.0.1
+acme_challenge_location: /etc/nginx/acme/
+
+servers:
+  your.domain.to_redirect:
+    redirect: "http://redirection.address"
+  your_other.doma.in:
+    redirect: "http://redirection-other.address"
+    server_options:
+      - "proxy_buffering off;"
+      - "proxy_request_buffering off;"
+    location_options:
+      - "proxy_http_version 1.1;"
+      - "proxy_set_header Upgrade $http_upgrade;"
+      - 'proxy_set_header Connection "upgrade";'
+      - "proxy_read_timeout 86400;"
+  "*.doma.in":
+    all_names: "*.doma.in doma.in"
+    redirect: "http://full.subdomain.redirect"
+    server_options:
+      - "proxy_buffering off;"
+      - "proxy_request_buffering off;"
+      - "client_max_body_size 0;"

run_once.d-c.yml

@@ -1,32 +1,32 @@
-version: '3.4'
+version: '3.4'
-
+
-services:
+services:
-  certbot:
+  certbot:
-    container_name: "certbot_get_certificates"
+    container_name: "certbot_get_certificates"
-    build:
+    build:
-      context: ./certbot
+      context: ./certbot
-      args:
+      args:
-        EMAIL: ${EMAIL}
+        EMAIL: ${EMAIL}
-    volumes:    
+    volumes:    
-      - ssl:/ssl
+      - ssl:/ssl
-      - letsencrypt:/etc/letsencrypt
+      - letsencrypt:/etc/letsencrypt
-    command: ["/run_once"]
+    command: ["/run_once"]
-    depends_on: 
+    depends_on: 
-      - nginx
+      - nginx
-
+
-  nginx:  
+  nginx:  
-    container_name: "nginx-get-certificates"
+    container_name: "nginx-get-certificates"
-    build:
+    build:
-      context: ./nginx
+      context: ./nginx
-      dockerfile: get-certificates.Dockerfile
+      dockerfile: get-certificates.Dockerfile
-    volumes:
+    volumes:
-      - ssl:/ssl
+      - ssl:/ssl
-      - letsencrypt:/etc/letsencrypt
+      - letsencrypt:/etc/letsencrypt
-    ports:
+    ports:
-      - 80:80
+      - 80:80
-
+
-volumes:
+volumes:
-  ssl:
+  ssl:
-    name: ssl_nginx_ssl
+    name: ssl_nginx_ssl
-  letsencrypt:
+  letsencrypt:
-    name: ssl_nginx_letsencrypt
+    name: ssl_nginx_letsencrypt