fix: 处理PG 连接泄漏隐患
This commit is contained in:
parent
e77c1ac820
commit
e5bf9d58c0
|
|
@ -1,7 +1,7 @@
|
||||||
import re
|
import re
|
||||||
import psycopg2
|
import psycopg2
|
||||||
import threading
|
import threading
|
||||||
from django.db import transaction
|
from django.db import transaction, connections
|
||||||
from .models import Message
|
from .models import Message
|
||||||
|
|
||||||
# 数据库连接
|
# 数据库连接
|
||||||
|
|
@ -190,6 +190,10 @@ def strip_sql_markdown(content: str) -> str:
|
||||||
# ORM 写入包装函数
|
# ORM 写入包装函数
|
||||||
def save_message_thread_safe(**kwargs):
|
def save_message_thread_safe(**kwargs):
|
||||||
def _save():
|
def _save():
|
||||||
|
try:
|
||||||
with transaction.atomic():
|
with transaction.atomic():
|
||||||
Message.objects.create(**kwargs)
|
Message.objects.create(**kwargs)
|
||||||
|
finally:
|
||||||
|
# 子线程退出前关闭本线程的 Django DB 连接,避免 PG 连接泄漏
|
||||||
|
connections.close_all()
|
||||||
threading.Thread(target=_save).start()
|
threading.Thread(target=_save).start()
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import threading
|
import threading
|
||||||
from apps.utils.decorators import auto_log
|
from apps.utils.decorators import auto_log
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from django.db import connections
|
||||||
|
|
||||||
# 创建全局线程池
|
# 创建全局线程池
|
||||||
global_executor = ThreadPoolExecutor(max_workers=20)
|
global_executor = ThreadPoolExecutor(max_workers=20)
|
||||||
|
|
@ -8,7 +9,12 @@ class MyThread(threading.Thread):
|
||||||
|
|
||||||
@auto_log('MyThread', raise_exception=True, send_mail=True)
|
@auto_log('MyThread', raise_exception=True, send_mail=True)
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
|
# 子线程退出 / 池内 worker 跑完一次任务后必须关闭本线程的 Django DB 连接,
|
||||||
|
# 否则 psycopg2 连接会一直驻留在线程的 thread-local,导致 PG "too many clients"
|
||||||
|
try:
|
||||||
return super().run()
|
return super().run()
|
||||||
|
finally:
|
||||||
|
connections.close_all()
|
||||||
|
|
||||||
def start_p(self):
|
def start_p(self):
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ django.setup()
|
||||||
|
|
||||||
from apps.enm.services import insert_mplogx_item
|
from apps.enm.services import insert_mplogx_item
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
from django.db import connections
|
||||||
from apps.utils.tasks import send_mail_task
|
from apps.utils.tasks import send_mail_task
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
CUR_DIR = os.path.dirname(os.path.abspath(__file__))
|
CUR_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
|
@ -92,6 +93,7 @@ def fetch_data(timex, enp_mpoint_dict, path):
|
||||||
"""
|
"""
|
||||||
从数据库转存到超表
|
从数据库转存到超表
|
||||||
"""
|
"""
|
||||||
|
try:
|
||||||
response = None
|
response = None
|
||||||
try:
|
try:
|
||||||
response = requests.get(path, timeout=5)
|
response = requests.get(path, timeout=5)
|
||||||
|
|
@ -118,6 +120,9 @@ def fetch_data(timex, enp_mpoint_dict, path):
|
||||||
current_object = [] # 重置,准备处理下一个对象
|
current_object = [] # 重置,准备处理下一个对象
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
send_error_notification(e)
|
send_error_notification(e)
|
||||||
|
finally:
|
||||||
|
# 子线程必须主动关闭 Django DB 连接,否则每分钟泄漏 2 条 PG 连接
|
||||||
|
connections.close_all()
|
||||||
|
|
||||||
def get_data():
|
def get_data():
|
||||||
last_triggered = None
|
last_triggered = None
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue