From 16bc1c0db75d3894fefe34c6fe2ef419c09cf992 Mon Sep 17 00:00:00 2001 From: kanootoko Date: Mon, 22 Apr 2024 17:58:41 +0300 Subject: [PATCH] update 2024-04-22 Changes: - add jinja2 template engine instead of string-replacing - fix certbot cron usage - replace json servers configuration with yaml --- .env.example | 2 +- .gitignore | 3 +- certbot/Dockerfile | 12 +- certbot_manual.sh | 4 +- docker-compose.yml | 78 +++++----- nginx/Dockerfile | 16 +-- nginx/add_servers.py | 228 ++++++++++++------------------ nginx/domains.txt.example | 6 +- nginx/get-certificates.Dockerfile | 4 +- nginx/nginx.conf | 27 ---- nginx/nginx.conf.j2 | 82 +++++++++++ nginx/proxy_common.conf | 3 - nginx/server_common.conf | 6 - nginx/servers.json.example | 8 -- nginx/servers.yaml.example | 23 +++ run_once.d-c.yml | 64 ++++----- 16 files changed, 292 insertions(+), 274 deletions(-) delete mode 100644 nginx/nginx.conf create mode 100644 nginx/nginx.conf.j2 delete mode 100644 nginx/proxy_common.conf delete mode 100644 nginx/server_common.conf delete mode 100644 nginx/servers.json.example create mode 100644 nginx/servers.yaml.example diff --git a/.env.example b/.env.example index 212b5b9..ad86344 100644 --- a/.env.example +++ b/.env.example @@ -1 +1 @@ -EMAIL=your@email.com \ No newline at end of file +EMAIL=your@email.com diff --git a/.gitignore b/.gitignore index afe4e47..8606e39 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /nginx/domains.txt /nginx/servers.json -*.env \ No newline at end of file +/nginx/servers.yaml +*.env diff --git a/certbot/Dockerfile b/certbot/Dockerfile index 114f9bc..6b7c639 100644 --- a/certbot/Dockerfile +++ b/certbot/Dockerfile @@ -6,15 +6,11 @@ ARG EMAIL RUN apk add certbot bash -RUN echo "#!/bin/sh" > /usr/bin/update_certificates && \ - echo "certbot renew --quiet" >> /usr/bin/update_certificates && \ - echo "cp -rL /etc/letsencrypt/live/* /ssl/" >> /usr/bin/update_certificates && \ +RUN mkdir -p /etc/letsencrypt /etc/letsencrypt.bak /ssl/.well-known && \ \ - mkdir -p /etc/letsencrypt /ssl/.well-known && \ + echo "webroot-path = /ssl/" > /etc/letsencrypt.bak/cli.ini && \ \ - echo "webroot-path = /ssl/" > /etc/letsencrypt/cli.ini && \ - \ - echo '15 2 */7 * * /usr/bin/update_certificates' > /etc/crontabs/root && \ + echo '15 2 */7 * * /run_once' > /etc/crontabs/root && \ \ echo "echo 'running with cron'" > /run_with_cron && \ echo "cp /etc/letsencrypt.bak/cli.ini /etc/letsencrypt/cli.ini" >> /run_with_cron && \ @@ -25,7 +21,7 @@ RUN echo "#!/bin/sh" > /usr/bin/update_certificates && \ echo "if [ ! -f /ssl/domains.txt ]; then echo 'No domains.txt file found in /ssl, exiting' && exit 1; fi" >> /run_once && \ echo 'for domain in $(cat /ssl/domains.txt); do case $domain in "#"*) :; ;; *) certbot certonly -n --authenticator webroot -d $domain; ;; esac; done' >> /run_once && \ echo "cp -rL /etc/letsencrypt/live/* /ssl/" >> /run_once && \ - chmod +x /usr/bin/update_certificates + chmod +x /run_once RUN certbot register --email $EMAIL --non-interactive --agree-tos diff --git a/certbot_manual.sh b/certbot_manual.sh index 42419d6..f6564af 100755 --- a/certbot_manual.sh +++ b/certbot_manual.sh @@ -8,7 +8,7 @@ if [ "$EMAIL" = '' ]; then fi docker build --tag certbot_manual_test --build-arg EMAIL="$EMAIL" certbot -echo "ececute 'cat /run_once' to get commands list" +echo "execute 'cat /run_once' to get commands list" echo "for wildcard domains use manual mode with dns challange: 'certbot certonly -d '*.domain' --manual --preferred-challenges dns" docker run -it --rm -v $SSL_VOLUME_NAME:/ssl -v $LETSENCRYPT_VOLUME_NAME:/etc/letsencrypt --entrypoint /bin/bash certbot_manual_test -docker rmi certbot_manual_test \ No newline at end of file +docker rmi certbot_manual_test diff --git a/docker-compose.yml b/docker-compose.yml index acfecda..205713f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,40 +1,38 @@ -version: '3.4' - -name: ssl_nginx - -services: - certbot: - container_name: 'global-certbot' - build: - context: ./certbot - args: - EMAIL: ${EMAIL} - volumes: - - ssl:/ssl - - letsencrypt:/etc/letsencrypt - depends_on: - - nginx - restart: unless-stopped - - nginx: - container_name: 'global-nginx' - build: ./nginx - volumes: - - ssl:/ssl - - letsencrypt:/etc/letsencrypt - ports: - - 80:80 - - 443:443 - networks: - - hosting_net - restart: unless-stopped - -volumes: - ssl: - name: ssl_nginx_ssl - letsencrypt: - name: ssl_nginx_letsencrypt - -networks: - hosting_net: - external: true +name: ssl_nginx + +services: + certbot: + container_name: 'global-certbot' + build: + context: ./certbot + args: + EMAIL: ${EMAIL} + volumes: + - ssl:/ssl + - letsencrypt:/etc/letsencrypt + depends_on: + - nginx + restart: unless-stopped + + nginx: + container_name: 'global-nginx' + build: ./nginx + volumes: + - ssl:/ssl + - letsencrypt:/etc/letsencrypt + ports: + - 80:80 + - 443:443 + networks: + - hosting_net + restart: unless-stopped + +volumes: + ssl: + name: ssl_nginx_ssl + letsencrypt: + name: ssl_nginx_letsencrypt + +networks: + hosting_net: + external: true diff --git a/nginx/Dockerfile b/nginx/Dockerfile index dc824db..03a2fd3 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -1,22 +1,22 @@ -FROM python:3.10-alpine as builder +FROM python:3.11-alpine as builder + +RUN pip3 install --no-cache-dir pyyaml jinja2 COPY add_servers.py /add_servers.py -COPY servers.json /servers.json +COPY servers.yaml /servers.yaml COPY domains.txt /domains.txt -COPY nginx.conf /nginx.conf +COPY nginx.conf.j2 /nginx.conf.j2 -RUN python /add_servers.py --nginx /nginx.conf --domains_list_txt /domains.txt --servers_config_json servers.json --certificates_path /ssl +RUN python /add_servers.py --nginx-template /nginx.conf.j2 -o /nginx.conf --domains_list_txt /domains.txt --servers-config servers.yaml --certificates-path /ssl +# service FROM nginx:alpine -COPY proxy_common.conf /etc/nginx/proxy_common.conf -COPY server_common.conf /etc/nginx/server_common.conf - COPY --from=builder /nginx.conf /etc/nginx/nginx.conf COPY domains.txt /domains.txt -RUN echo "16 2 */7 * * nginx -s reload" > /etc/crontabs/certbot && \ +RUN echo "16 2 */7 * * nginx -s reload" > /etc/crontabs/root && \ \ echo "cp /domains.txt /ssl/domains.txt" > /entrypoint && \ echo "crond" >> /entrypoint && \ diff --git a/nginx/add_servers.py b/nginx/add_servers.py index 29851d3..4f3f98c 100644 --- a/nginx/add_servers.py +++ b/nginx/add_servers.py @@ -1,121 +1,87 @@ """Add domain servers to nginx.conf executable script.""" +from __future__ import annotations + import argparse +import os from dataclasses import dataclass, field -import json -import sys +from typing import Any -REDIRECT_TEMPLATE = "\n".join( - ( - "\tlocation / {{", - "\t\tresolver 127.0.0.11 valid=30s;", - "\t\tset $host_{i} {redirection_host};", - "\t\tproxy_pass $host_{i}/;", - "\t\tinclude proxy_common.conf;", - "\t}}", - ) -) - -SERVER_HTTPS_TEMPLATE = "\n".join( - ( - "server {{", - "\tinclude server_common.conf;", - "\tlisten 443 ssl;", - "\t", - "\tserver_name {server_name};", - "\tssl_certificate {certificates_path}/{certificate}/fullchain.pem;", - "\tssl_certificate_key {certificates_path}/{certificate}/privkey.pem;", - "\t", - "{options}", - "{redirection}", - "}}", - ) -) - -SERVER_HTTP_TEMPLATE = "\n".join( - ( - "server {{", - "\tinclude server_common.conf;", - "\t", - "\tserver_name {server_name};", - "{options}", - "{redirection}", - "}}", - ) -) +import jinja2 +import yaml @dataclass class Server: """Server entry for nginx.conf file.""" - server_name: str + name: str + all_names: str | None = None redirect: str | None = None - certificate: str | None = None - options: list[str] = field(default_factory=list) + certificate_dir: str | None = None + port: int = None + ssl_port: int = None + server_options: list[str] = field(default_factory=list) + location_options: list[str] = field(default_factory=list) certificates_path: str = "/etc/letsencrypt/live" def __post_init__(self) -> None: - if self.options is None: - self.options = [] + if self.server_options is None: + self.server_options = [] + if self.location_options is None: + self.location_options = [] + if self.all_names is None: + self.all_names = self.name + if self.port is None: + self.port = 80 + if self.ssl_port is None: + self.ssl_port = 443 - def format(self, i: int, indent: str = " ", base_indent: int = 1) -> str: - """Format server to place inside nginx.conf""" - res = (indent * base_indent) + ( - SERVER_HTTPS_TEMPLATE if self.certificate is not None else SERVER_HTTP_TEMPLATE - ).replace("\n", "\n" + indent * base_indent).replace("\t", indent).format( - server_name=self.server_name, - certificate=self.certificate, - redirection=( - REDIRECT_TEMPLATE.replace("\n", "\n" + indent * base_indent) - .replace("\t", indent) - .format(i=i, redirection_host=self.redirect) - if self.redirect is not None - else "" - ), - options=( - ("\n" + indent * (base_indent + 1)) - + ("\n" + indent * (base_indent + 1)).join( - ("\n" + indent * (base_indent + 1)).join(option.split("\n")) for option in self.options - ) - ), - certificates_path=self.certificates_path, - ) - res = "\n".join(line for line in res.split("\n") if line.strip() != "") - return res + def to_dict(self) -> dict: + """Convert server class to dict for nginx.conf.j2 jinja2-template""" + return { + "name": self.name, + "all_names": self.all_names, + "redirect": self.redirect, + "server_options": self.server_options, + "location_options": self.location_options, + "certificate_dir": self.certificate_dir, + "port": self.port, + "ssl_port": self.ssl_port, + } @dataclass class CLIParams: """add_servers CLI parameters""" - nginx: str + nginx_template: str domains_list_txt: str - servers_config_json: str + servers_config: str certificates_path: str http_only: bool - dry_run: bool + output: str | None def main() -> None: """Parse arguments and add domains to nginx.conf""" parser = argparse.ArgumentParser("add-servers", description="Add domain servers to a given nginx.conf file") - parser.add_argument("--nginx", "-f", required=True, help="Path to nginx.conf file to edit inplace") + parser.add_argument("--nginx-template", "-f", required=True, help="Path to nginx.conf.j2 template file") parser.add_argument( "--domains_list_txt", "-d", required=True, - help="Path to domains list with ssl certificates ", + help="Path to file with domains which have ssl certificates", ) parser.add_argument( - "--servers_config_json", + "--servers-config", "-s", required=False, default=None, - help='Path to domains json {"domain": {"redirect": "redirection_path", "options": []}};', + help="Path to servers configuration yaml file", ) parser.add_argument( - "--certificates_path", + "--certificates-path", "-c", required=False, default="/etc/letsencrypt/live", @@ -126,85 +92,77 @@ def main() -> None: action="store_true", help="Remove certificates usage from servers section", ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Print servers section and quit withot making changes", - ) + parser.add_argument("--output", "-o", help="Path to nginx.conf output file") args: CLIParams = parser.parse_args() - replacement_substring = "# " - - with open(args.nginx, "r", encoding="utf-8") as file: - nginx_config = file.read() - if nginx_config.count(replacement_substring) != 1: - print( - f"Error. File must contain exactly one '{replacement_substring}' substring to replace with servers. Exiting" - ) - sys.exit(1) - if args.domains_list_txt is not None: with open(args.domains_list_txt, "r", encoding="utf-8") as file: - domains_certs_list = [ + domains_with_certs = [ domain.strip() for domain in file.readlines() if domain.strip() != "" and not domain.startswith("#") ] else: - domains_certs_list = [] + domains_with_certs = [] nginx_servers: list[Server] = [] - if args.servers_config_json is not None: - with open(args.servers_config_json, "r", encoding="utf-8") as file: - for server_name, params in json.load(file).items(): - server_name: str - params: dict[str, str] - nginx_servers.append( - Server( - (server_name if "*" not in server_name else f"{server_name} {server_name.replace('*.', '')}"), - params.get("redirect"), - ( - None - if args.http_only - else server_name.replace("*.", "") - if server_name in domains_certs_list - else server_name[server_name.find(".") + 1 :] - if "." in server_name - and f"*.{server_name[server_name.find('.') + 1:]}" in domains_certs_list - else None - ), - params.get("options"), - args.certificates_path, - ) + if args.servers_config is not None: + with open(args.servers_config, "r", encoding="utf-8") as file: + data: dict = yaml.safe_load(file) + resolver: str = data.get("resolver", "127.0.0.1") + acme_challenge_location: str | None = data.get("acme_challenge_location") + servers: dict[str, dict[str, Any]] = data["servers"] + + for server_name, params in servers.items(): + nginx_servers.append( + Server( + name=(server_name if "*" not in server_name else f"{server_name} {server_name.replace('*.', '')}"), + all_names=params.get("all_names"), + redirect=params.get("redirect"), + certificate_dir=_get_certificate_path( + args.http_only, domains_with_certs, args.certificates_path, server_name + ), + port=params.get("port"), + ssl_port=params.get("ssl_port"), + server_options=params.get("server_options"), + location_options=params.get("location_options"), ) - for domain in domains_certs_list: + ) + for domain in domains_with_certs: if not any( ( - domain == server.server_name - or ( - domain == f"*.{server.server_name[server.server_name.find('.') + 1:]}" - if "." in server.server_name - else False - ) + domain == server.name + or (domain == f"*.{server.name[server.name.find('.') + 1:]}" if "." in server.name else False) ) for server in nginx_servers ): - nginx_servers.append( - Server(domain, None, domain if not args.http_only else None, certificates_path=args.certificates_path) - ) + nginx_servers.append(Server(name=domain)) - nginx_servers_part = "\n\n".join(server.format(i) for i, server in enumerate(nginx_servers)) + with open(args.nginx_template, "r", encoding="utf-8") as file: + template = jinja2.Environment().from_string(file.read()) - print(f"Using following servers part for nginx.conf:\n\n{nginx_servers_part}") + result = template.render( + resolver=resolver, + acme_challenge_location=acme_challenge_location, + servers=[server.to_dict() for server in nginx_servers], + ) - if not args.dry_run: - config_backup_filename = f"{args.nginx}.bak" - print(f"Backing up old nginx config {args.nginx} as {config_backup_filename}") + print(result) - with open(config_backup_filename, "w", encoding="utf-8") as file: - file.write(nginx_config) + if args.output is not None: + with open(args.output, "w", encoding="utf-8") as file: + file.write(result) - with open(args.nginx, "w", encoding="utf-8") as file: - file.write(nginx_config.replace(replacement_substring, nginx_servers_part.lstrip())) + +def _get_certificate_path( + http_only: bool, domains_with_certs: list[str], base_certs_path: str, server_name: str +) -> str | None: + if http_only: + return None + if server_name in domains_with_certs: + return os.path.join(base_certs_path, server_name.replace("*.", "")) + if "." in server_name and f"*.{server_name[server_name.find('.') + 1:]}" in domains_with_certs: + return os.path.join(base_certs_path, server_name[server_name.find(".") + 1 :]) + return None if __name__ == "__main__": diff --git a/nginx/domains.txt.example b/nginx/domains.txt.example index e39f4e1..e3488c9 100644 --- a/nginx/domains.txt.example +++ b/nginx/domains.txt.example @@ -1,3 +1,5 @@ your.domain.to_redirect -your.other_domain -#commented.domain \ No newline at end of file +your_other.doma.in +*.doma.in +#commented.domain +domain.without.redirect diff --git a/nginx/get-certificates.Dockerfile b/nginx/get-certificates.Dockerfile index 081a6a7..00eb5cb 100644 --- a/nginx/get-certificates.Dockerfile +++ b/nginx/get-certificates.Dockerfile @@ -6,6 +6,8 @@ COPY nginx.conf /nginx.conf RUN python /add_servers.py --nginx /nginx.conf --domains_list_txt /domains.txt --http-only +# service + FROM nginx:alpine COPY proxy_common.conf /etc/nginx/proxy_common.conf @@ -14,7 +16,7 @@ COPY server_common.conf /etc/nginx/server_common.conf COPY --from=builder /nginx.conf /etc/nginx/nginx.conf COPY domains.txt /domains.txt -RUN echo "(sleep 60 && killall nginx) &" > /entrypoint && \ +RUN echo "(sleep 120 && killall nginx) &" > /entrypoint && \ echo "cp /domains.txt /ssl/domains.txt" >> /entrypoint && \ echo "nginx -g 'daemon off;'" >> /entrypoint && \ echo "nginx" >> /entrypoint diff --git a/nginx/nginx.conf b/nginx/nginx.conf deleted file mode 100644 index 3eeee77..0000000 --- a/nginx/nginx.conf +++ /dev/null @@ -1,27 +0,0 @@ -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log notice; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 10m; - server_tokens off; - gzip on; - proxy_connect_timeout 300; - proxy_send_timeout 600; - proxy_read_timeout 600; - send_timeout 600; - client_max_body_size 1100M; - - server { - return 404; - } - - # -} diff --git a/nginx/nginx.conf.j2 b/nginx/nginx.conf.j2 new file mode 100644 index 0000000..6d191eb --- /dev/null +++ b/nginx/nginx.conf.j2 @@ -0,0 +1,82 @@ +{#- variables ~ examples: #} +{#- acme_challenge_location ~ /ssl/: #} +{#- resolver ~ 127.0.0.11: #} +{#- servers ~ ["name": ..., ("redirect": ..., "server_options": ..., "location_options": ..., "all_names": ..., "port": ..., "ssl_port": ...)]: #} +{#- #}user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + server_tokens off; + gzip on; + + proxy_connect_timeout 300; + proxy_send_timeout 600; + proxy_read_timeout 600; + send_timeout 600; + client_max_body_size 500M; + + server { + return 404; + } + + {%- for server in servers %} + server { + listen {{ server["port"] or 80 }}; + {%- if server["certificate_dir"] is not none %} + listen {{ server["ssl_port"] or 443 }} ssl; + {%- endif %} + keepalive_timeout 70; + + server_name {{ server["all_names"] or server["name"] }}; + + {%- if acme_challenge_location is defined %} + {# #} + location /.well-known/acme-challenge { + root {{ acme_challenge_location }}; + } + {%- endif %} + + {%- if server["server_options"]|length > 0 %} + {# #} + {%- for server_option in server["server_options"] %} + {{ server_option }} + {%- endfor %} + {%- endif %} + + {%- if server["certificate_dir"] is not none %} + {# #} + ssl_certificate {{ server["certificate_dir"] }}/fullchain.pem; + ssl_certificate_key {{ server["certificate_dir"] }}/privkey.pem; + {%- endif %} + + {%- if server["redirect"] is not none %} + {# #} + location / { + resolver {{ resolver }}; + set $host_{{ loop.index }} {{ server["redirect"] }}; + proxy_pass $host_{{ loop.index }}; + + proxy_set_header Host $host:$server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + + {%- if server["location_options"]|length > 0 %} + {# #} + {%- for location_option in server["location_options"] %} + {{ location_option }} + {%- endfor %} + {%- endif %} + } + {%- endif %} + } + {%- endfor %} +} diff --git a/nginx/proxy_common.conf b/nginx/proxy_common.conf deleted file mode 100644 index 41f4005..0000000 --- a/nginx/proxy_common.conf +++ /dev/null @@ -1,3 +0,0 @@ - proxy_set_header Host $host:$server_port; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; diff --git a/nginx/server_common.conf b/nginx/server_common.conf deleted file mode 100644 index 036cf8a..0000000 --- a/nginx/server_common.conf +++ /dev/null @@ -1,6 +0,0 @@ - listen 80; - keepalive_timeout 70; - - location /.well-known/acme-challenge { - root /ssl/; - } diff --git a/nginx/servers.json.example b/nginx/servers.json.example deleted file mode 100644 index dd5fa79..0000000 --- a/nginx/servers.json.example +++ /dev/null @@ -1,8 +0,0 @@ -{ - "server.name": { - "redirect": "http://local.address", - "options": [ - "other_server_level_nginx_options here;" - ] - } -} \ No newline at end of file diff --git a/nginx/servers.yaml.example b/nginx/servers.yaml.example new file mode 100644 index 0000000..2db8554 --- /dev/null +++ b/nginx/servers.yaml.example @@ -0,0 +1,23 @@ +resolver: 127.0.0.1 +acme_challenge_location: /etc/nginx/acme/ + +servers: + your.domain.to_redirect: + redirect: "http://redirection.address" + your_other.doma.in: + redirect: "http://redirection-other.address" + server_options: + - "proxy_buffering off;" + - "proxy_request_buffering off;" + location_options: + - "proxy_http_version 1.1;" + - "proxy_set_header Upgrade $http_upgrade;" + - 'proxy_set_header Connection "upgrade";' + - "proxy_read_timeout 86400;" + "*.doma.in": + all_names: "*.doma.in doma.in" + redirect: "http://full.subdomain.redirect" + server_options: + - "proxy_buffering off;" + - "proxy_request_buffering off;" + - "client_max_body_size 0;" diff --git a/run_once.d-c.yml b/run_once.d-c.yml index 8df8406..d68047c 100644 --- a/run_once.d-c.yml +++ b/run_once.d-c.yml @@ -1,32 +1,32 @@ -version: '3.4' - -services: - certbot: - container_name: "certbot_get_certificates" - build: - context: ./certbot - args: - EMAIL: ${EMAIL} - volumes: - - ssl:/ssl - - letsencrypt:/etc/letsencrypt - command: ["/run_once"] - depends_on: - - nginx - - nginx: - container_name: "nginx-get-certificates" - build: - context: ./nginx - dockerfile: get-certificates.Dockerfile - volumes: - - ssl:/ssl - - letsencrypt:/etc/letsencrypt - ports: - - 80:80 - -volumes: - ssl: - name: ssl_nginx_ssl - letsencrypt: - name: ssl_nginx_letsencrypt +version: '3.4' + +services: + certbot: + container_name: "certbot_get_certificates" + build: + context: ./certbot + args: + EMAIL: ${EMAIL} + volumes: + - ssl:/ssl + - letsencrypt:/etc/letsencrypt + command: ["/run_once"] + depends_on: + - nginx + + nginx: + container_name: "nginx-get-certificates" + build: + context: ./nginx + dockerfile: get-certificates.Dockerfile + volumes: + - ssl:/ssl + - letsencrypt:/etc/letsencrypt + ports: + - 80:80 + +volumes: + ssl: + name: ssl_nginx_ssl + letsencrypt: + name: ssl_nginx_letsencrypt