Merge branch 'master' into develop

backend/application/celery.py

@@ -1,9 +1,11 @@
+import functools
 import os
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'application.settings')
 
 from django.conf import settings
 from celery import platforms
+
 # 租户模式
 if "django_tenants" in settings.INSTALLED_APPS:
     from tenant_schemas_celery.app import CeleryApp as TenantAwareCeleryApp
@@ -16,3 +18,23 @@ else:
 app.config_from_object('django.conf:settings')
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 platforms.C_FORCE_ROOT = True
+
+
+def retry_base_task_error():
+    """
+    celery 失败重试装饰器
+    :return:
+    """
+
+    def wraps(func):
+        @app.task(bind=True, retry_delay=180, max_retries=3)
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as exc:
+                raise self.retry(exc=exc)
+
+        return wrapper
+
+    return wraps

backend/application/settings.py

@@ -63,6 +63,7 @@ INSTALLED_APPS = [
 ]
 
 MIDDLEWARE = [
+    "dvadmin.utils.middleware.HealthCheckMiddleware",
     "django.middleware.security.SecurityMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",

backend/dvadmin/utils/middleware.py

@@ -2,9 +2,11 @@
 日志 django中间件
 """
 import json
+import logging
 
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
+from django.http import HttpResponse, HttpResponseServerError
 from django.utils.deprecation import MiddlewareMixin
 
 from dvadmin.system.models import OperationLog
@@ -87,3 +89,58 @@ class ApiLoggingMiddleware(MiddlewareMixin):
             if self.methods == 'ALL' or request.method in self.methods:
                 self.__handle_response(request, response)
         return response
+
+logger = logging.getLogger("healthz")
+class HealthCheckMiddleware(object):
+    """
+    存活检查中间件
+    """
+    def __init__(self, get_response):
+        self.get_response = get_response
+        # One-time configuration and initialization.
+
+    def __call__(self, request):
+        if request.method == "GET":
+            if request.path == "/readiness":
+                return self.readiness(request)
+            elif request.path == "/healthz":
+                return self.healthz(request)
+        return self.get_response(request)
+
+    def healthz(self, request):
+        """
+        Returns that the server is alive.
+        """
+        return HttpResponse("OK")
+
+    def readiness(self, request):
+        # Connect to each database and do a generic standard SQL query
+        # that doesn't write any data and doesn't depend on any tables
+        # being present.
+        try:
+            from django.db import connections
+            for name in connections:
+                cursor = connections[name].cursor()
+                cursor.execute("SELECT 1;")
+                row = cursor.fetchone()
+                if row is None:
+                    return HttpResponseServerError("db: invalid response")
+        except Exception as e:
+            logger.exception(e)
+            return HttpResponseServerError("db: cannot connect to database.")
+
+        # Call get_stats() to connect to each memcached instance and get it's stats.
+        # This can effectively check if each is online.
+        try:
+            from django.core.cache import caches
+            from django.core.cache.backends.memcached import BaseMemcachedCache
+            for cache in caches.all():
+                if isinstance(cache, BaseMemcachedCache):
+                    stats = cache._cache.get_stats()
+                    if len(stats) != len(cache._servers):
+                        return HttpResponseServerError("cache: cannot connect to cache.")
+        except Exception as e:
+            logger.exception(e)
+            return HttpResponseServerError("cache: cannot connect to cache.")
+
+        return HttpResponse("OK")