feat: 新增翁福(KIO)数据采集、FastAPI查询接口及位号映射

- mqttc: 支持KIO datachange偏移使能格式解析,翁福位号白名单映射,
  HTTP /stats统计接口,邮件报警限流,表名列名重构,MQTT认证
- api: 新增FastAPI数据查询服务(latest/history/mpoints/count/stats)
- load_mpoint_name: Excel位号导入脚本
- supervisor_api.conf: API服务supervisor配置

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GhsYr2ZyGxsGw7vnyuEA5Q
This commit is contained in:
TianyangZhang 2026-07-21 02:26:42 -04:00
parent 39a85b4d18
commit 42fc032ec1
4 changed files with 572 additions and 18 deletions

193
api.py Normal file
View File

@ -0,0 +1,193 @@
"""轻量数据查询接口,独立于 mqttc 进程运行。
启动: uvicorn api:app --host 0.0.0.0 --port 5900
"""
import os
import sys
CUR_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, CUR_DIR)
import conf
from fastapi import FastAPI, Query, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy import create_engine, text
from datetime import datetime, timedelta
from typing import Optional
engine = create_engine(conf.DATABASE_URL, pool_size=5, max_overflow=10, pool_pre_ping=True)
ALLOWED_TABLES = {
'mplogx', 'mplogx_xzzl', 'mplogx_tlxn',
'mplogx_hknf_l1', 'mplogx_hknf_l2', 'mplogx_hknf_l3',
'mplogx_wengfu',
}
app = FastAPI(title="MTC 数据查询接口", version="1.0.0")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_methods=["*"],
allow_headers=["*"],
)
@app.get("/api/latest")
def get_latest(
table: str = Query(..., description="表名,如 mplogx_wengfu"),
mpoint_id: str = Query(..., description="测点ID"),
limit: int = Query(1, ge=1, le=100),
):
"""获取指定测点的最新数据"""
if table not in ALLOWED_TABLES:
raise HTTPException(400, f"不允许的表名: {table}")
sql = text(f"""
SELECT timex, mpoint_id, val_float, val_str
FROM "{table}"
WHERE mpoint_id = :mpoint_id
ORDER BY timex DESC
LIMIT :limit
""")
with engine.connect() as conn:
rows = conn.execute(sql, {"mpoint_id": mpoint_id, "limit": limit}).fetchall()
return [
{"timex": r.timex.isoformat(), "mpoint_id": r.mpoint_id,
"val_float": r.val_float, "val_str": r.val_str}
for r in rows
]
@app.get("/api/history")
def get_history(
table: str = Query(..., description="表名"),
mpoint_id: str = Query(..., description="测点ID"),
start: str = Query(..., description="起始时间 yyyy-MM-dd HH:mm:ss"),
end: Optional[str] = Query(None, description="结束时间,默认当前"),
limit: int = Query(1000, ge=1, le=10000),
):
"""按时间范围查询测点历史数据"""
if table not in ALLOWED_TABLES:
raise HTTPException(400, f"不允许的表名: {table}")
try:
start_dt = datetime.strptime(start, "%Y-%m-%d %H:%M:%S")
except ValueError:
raise HTTPException(400, "start 格式错误,需要 yyyy-MM-dd HH:mm:ss")
end_dt = datetime.now()
if end:
try:
end_dt = datetime.strptime(end, "%Y-%m-%d %H:%M:%S")
except ValueError:
raise HTTPException(400, "end 格式错误,需要 yyyy-MM-dd HH:mm:ss")
sql = text(f"""
SELECT timex, mpoint_id, val_float, val_str
FROM "{table}"
WHERE mpoint_id = :mpoint_id
AND timex >= :start AND timex < :end
ORDER BY timex
LIMIT :limit
""")
with engine.connect() as conn:
rows = conn.execute(sql, {
"mpoint_id": mpoint_id, "start": start_dt, "end": end_dt, "limit": limit
}).fetchall()
return [
{"timex": r.timex.isoformat(), "mpoint_id": r.mpoint_id,
"val_float": r.val_float, "val_str": r.val_str}
for r in rows
]
@app.get("/api/mpoints")
def list_mpoints(
table: str = Query(..., description="表名"),
keyword: Optional[str] = Query(None, description="模糊搜索测点ID"),
limit: int = Query(200, ge=1, le=1000),
):
"""列出表内的测点ID去重"""
if table not in ALLOWED_TABLES:
raise HTTPException(400, f"不允许的表名: {table}")
if keyword:
sql = text(f"""
SELECT DISTINCT mpoint_id FROM "{table}"
WHERE mpoint_id LIKE :kw
ORDER BY mpoint_id LIMIT :limit
""")
params = {"kw": f"%{keyword}%", "limit": limit}
else:
sql = text(f"""
SELECT DISTINCT mpoint_id FROM "{table}"
ORDER BY mpoint_id LIMIT :limit
""")
params = {"limit": limit}
with engine.connect() as conn:
rows = conn.execute(sql, params).fetchall()
return [r.mpoint_id for r in rows]
@app.get("/api/count")
def get_count():
"""查询每个 mplogx 表的总条数"""
with engine.connect() as conn:
tables = conn.execute(text("""
SELECT tablename FROM pg_tables
WHERE schemaname = current_schema() AND tablename LIKE 'mplogx%'
ORDER BY tablename
""")).fetchall()
if not tables:
return {"tables": [], "total": 0}
results = []
total = 0
for (tablename,) in tables:
count = conn.execute(text(
f'SELECT COUNT(*) FROM "{tablename}"'
)).scalar()
results.append({"table_name": tablename, "row_count": count})
total += count
return {"tables": results, "total": total}
@app.get("/api/stats")
def get_stats():
"""各 mplogx 表的统计信息(复用 mqttc 中的逻辑)"""
with engine.connect() as conn:
tables = conn.execute(text("""
SELECT tablename FROM pg_tables
WHERE schemaname = current_schema() AND tablename LIKE 'mplogx%'
ORDER BY tablename
""")).fetchall()
if not tables:
return {"tables": [], "summary": {}}
parts = []
for (tablename,) in tables:
parts.append(f"""
SELECT '{tablename}' AS table_name,
COUNT(DISTINCT mpoint_id)::bigint AS point_count_1h,
approximate_row_count('{tablename}')::numeric AS approximate_row_count
FROM "{tablename}"
WHERE timex >= now() - interval '1 hour' AND timex < now()
""")
union_sql = "\nUNION ALL\n".join(parts)
rows = conn.execute(text(f"""
SELECT table_name, point_count_1h, approximate_row_count
FROM ({union_sql}) AS t ORDER BY table_name
""")).fetchall()
result = []
for table_name, point_count_1h, approx in rows:
result.append({
"table_name": table_name,
"point_count_1h": int(point_count_1h or 0),
"approximate_row_count": int(approx or 0),
})
total_points = sum(r["point_count_1h"] for r in result)
total_rows = sum(r["approximate_row_count"] for r in result)
return {
"tables": result,
"summary": {
"point_count_1h_total": total_points,
"approximate_row_count_total": total_rows,
"approximate_row_count_total_yi": round(total_rows / 100000000, 2),
},
}

101
load_mpoint_name.py Normal file
View File

@ -0,0 +1,101 @@
"""一次性脚本:从「翁福 数据远传变量.xlsx」解析点位建立并灌入 mpoint_name 映射表。
- 位号(mpoint_id) mqttc.code_from_name 切出与线上入库逻辑同一套规则杜绝漂移
- point_name = 全名去掉尾部位号full_name 保留 Excel 原文便于溯源
- 幂等ON CONFLICT(mpoint_id) DO UPDATE可反复执行
用法:
python load_mpoint_name.py [Excel路径] [数据库URL]
默认读取桌面上的数据远传变量.xlsx写入 conf.DATABASE_URL 指向的库
可传第二个参数指定目标库如线上 10.0.11.52
"""
import os
import sys
import openpyxl
from sqlalchemy import create_engine, text
CUR_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, CUR_DIR)
import conf
from mqttc import code_from_name
DEFAULT_XLSX = r"C:\Users\11825\Desktop\01-文档资料\资料\翁福采集\数据远传变量.xlsx"
TABLE = "wengfu_mpoint_name"
DDL = f"""
CREATE TABLE IF NOT EXISTS {TABLE} (
mpoint_id varchar PRIMARY KEY,
point_name text,
full_name text,
dtype varchar
)
"""
UPSERT = f"""
INSERT INTO {TABLE} (mpoint_id, point_name, full_name, dtype)
VALUES (:mpoint_id, :point_name, :full_name, :dtype)
ON CONFLICT (mpoint_id) DO UPDATE
SET point_name = EXCLUDED.point_name,
full_name = EXCLUDED.full_name,
dtype = EXCLUDED.dtype
"""
def read_items(path):
"""Excel 为左右两张并排的表:列 A/B 和 D/E 各是 (变量名, 类型)。"""
wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
ws = wb["Sheet1"]
items = []
for r in list(ws.iter_rows(values_only=True))[1:]: # 跳过表头
r = list(r)
if r and r[0]:
items.append((str(r[0]).strip(), str(r[1]).strip() if len(r) > 1 and r[1] else ''))
if len(r) > 3 and r[3]:
items.append((str(r[3]).strip(), str(r[4]).strip() if len(r) > 4 and r[4] else ''))
return items
def split_row(full_name, dtype):
code = code_from_name(full_name)
point = full_name[:len(full_name) - len(code)].rstrip('_') if code and full_name.endswith(code) else full_name
return {
"mpoint_id": code,
"point_name": point or None,
"full_name": full_name,
"dtype": dtype or None,
}
def main():
path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_XLSX
db_url = sys.argv[2] if len(sys.argv) > 2 else conf.DATABASE_URL
items = read_items(path)
rows = [split_row(name, typ) for name, typ in items]
# 位号唯一性自检
seen = {}
collisions = []
for row in rows:
mid = row["mpoint_id"]
if mid in seen and seen[mid] != row["full_name"]:
collisions.append((mid, seen[mid], row["full_name"]))
seen[mid] = row["full_name"]
if collisions:
print(f"!! 位号冲突 {len(collisions)} 处(不同全名切出同一位号),请先核对:")
for mid, a, b in collisions[:20]:
print(f" {mid}: {a} <> {b}")
sys.exit(1)
engine = create_engine(db_url, connect_args={'connect_timeout': 10})
with engine.begin() as conn:
conn.execute(text(DDL))
for row in rows:
conn.execute(text(UPSERT), row)
total = conn.execute(text(f"SELECT COUNT(*) FROM {TABLE}")).scalar()
print(f"目标库写入完成:读取 {len(items)} 条,去重后位号 {len(seen)} 个,{TABLE} 表内现有 {total} 行。")
if __name__ == '__main__':
main()

281
mqttc.py
View File

@ -1,17 +1,19 @@
import os import os
import sys import sys
import re
import paho.mqtt.client as mqtt import paho.mqtt.client as mqtt
import json import json
import logging import logging
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
from sqlalchemy import create_engine, Engine, text from sqlalchemy import create_engine, Engine, text
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from datetime import datetime from datetime import datetime, timedelta
from dateutil import tz from dateutil import tz
import smtplib import smtplib
from email.mime.multipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
from threading import Thread from threading import Thread, Lock
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import traceback import traceback
import queue import queue
@ -36,20 +38,47 @@ engine:Engine = None
# {table_name: {mpoint_id: last_timex}} 按表分桶,避免跨表互相抑制 # {table_name: {mpoint_id: last_timex}} 按表分桶,避免跨表互相抑制
mpoint_dict: dict = {} mpoint_dict: dict = {}
SHANGHAI = tz.gettz('Asia/Shanghai')
# topic -> 目标表名 映射 # topic -> 目标表名 映射
TOPIC_TABLE_MAP = { TOPIC_TABLE_MAP = {
conf.MQTT_TOPIC: 'mplogx', conf.MQTT_TOPIC: 'mplogx',
conf.MQTT_TOPIC_zl: 'mplogx_zlwd', conf.MQTT_TOPIC_zl: 'mplogx_xzzl',
conf.MQTT_TOPIC_tl: 'mplogx_tlsn', conf.MQTT_TOPIC_tl: 'mplogx_tlxn',
conf.MQTT_TOPIC_1: 'mplogx_hknf_l1', conf.MQTT_TOPIC_1: 'mplogx_hknf_l1',
conf.MQTT_TOPIC_2: 'mplogx_hknf_l2', conf.MQTT_TOPIC_2: 'mplogx_hknf_l2',
conf.MQTT_TOPIC_3: 'mplogx_hknf_l3', conf.MQTT_TOPIC_3: 'mplogx_hknf_l3',
conf.MQTT_TOPIC_kio: 'mplogx_wengfu',
} }
# KIO datachange 格式(偏移使能)的 topic走独立解析逻辑
KIO_TOPICS = {conf.MQTT_TOPIC_kio}
# 各表 INSERT 用的值列(除 timex, mpoint_id 外);未列出的表用默认两列
# mplogx_wengfu(KIO) 按值类型分列存储bool->val_bool, int->val_int, float->val_float, 文本->val_str
TABLE_VALUE_COLS = {
'mplogx_mqtt': ['val_float', 'val_str', 'val_q'],
'mplogx_wengfu': ['val_float', 'val_str', 'val_q', 'val_bool', 'val_int'],
}
DEFAULT_VALUE_COLS = ['val_float', 'val_str']
# topic -> 独立队列,每个 topic 一个 worker 线程消费 # topic -> 独立队列,每个 topic 一个 worker 线程消费
msg_queues: dict = {topic: queue.Queue() for topic in TOPIC_TABLE_MAP} msg_queues: dict = {topic: queue.Queue() for topic in TOPIC_TABLE_MAP}
# 错误邮件发送限流:最多 1 小时发一封
EMAIL_INTERVAL = 3600
_last_email_time = 0
_email_lock = Lock()
def send_error_email(message, subject='hfnf_mqtt', ): def send_error_email(message, subject='hfnf_mqtt', ):
now = time.time()
with _email_lock:
global _last_email_time
if now - _last_email_time < EMAIL_INTERVAL:
logger.info(f"距上次报警邮件不足 {EMAIL_INTERVAL}s跳过本次发送: {message}")
return
_last_email_time = now
msg = MIMEMultipart() msg = MIMEMultipart()
msg['From'] = conf.EMAIL_HOST_USER msg['From'] = conf.EMAIL_HOST_USER
msg['To'] = conf.EMAIL_HOST_USER msg['To'] = conf.EMAIL_HOST_USER
@ -72,7 +101,11 @@ def worker(topic: str):
try: try:
payload = q.get(timeout=60) payload = q.get(timeout=60)
if payload is not None: if payload is not None:
save_items(topic, payload) try:
save_items(topic, payload)
except Exception as e:
logger.error(f"[{table_name}] save_items 异常: {e}", exc_info=True)
Thread(target=send_error_email, args=(f"[{table_name}] save_items 异常: {e}",)).start()
last_message_time = time.time() last_message_time = time.time()
last_alert_time = 0 last_alert_time = 0
@ -85,31 +118,159 @@ def worker(topic: str):
).start() ).start()
last_alert_time = now last_alert_time = now
def normalize_standard(payload):
"""现有 6 个 topic 的格式 [{name,value,time,quality?}] -> [(timex, mpoint_id, value, quality)]"""
records = []
for item in json.loads(payload):
timex = datetime.strptime(item['time'], "%Y%m%d%H%M%S").replace(tzinfo=SHANGHAI)
records.append((timex, item['name'], item.get('value'), item.get('quality')))
return records
# 翁福(KIO)白名单映射:{点位全名 N -> 位号},启动时从 wengfu_mpoint_name 加载。
# 只有命中白名单的点位才把 mpoint_id 缩短为位号;其余(裸位号点、以及前/后位
# 限位这类尾串无区分度、缩短会串台的点)一律原样入库,避免不同物理点碰撞合并。
wengfu_name_map: dict = {}
def load_wengfu_name_map(eng):
"""从 wengfu_mpoint_name 载入 full_name -> mpoint_id 映射。失败不影响主流程。"""
global wengfu_name_map
try:
with eng.connect() as conn:
rows = conn.execute(text(
"SELECT full_name, mpoint_id FROM wengfu_mpoint_name WHERE full_name IS NOT NULL"
)).fetchall()
wengfu_name_map = {full: mid for full, mid in rows}
logger.info(f"载入翁福位号映射 {len(wengfu_name_map)}")
except Exception as e:
logger.error(f"载入 wengfu_mpoint_name 失败,本次将按原样存全名: {e}", exc_info=True)
wengfu_name_map = {}
def code_from_name(name):
"""从翁福(KIO)点位全名切出位号(供建表脚本 load_mpoint_name.py 生成白名单用)。
运行时入库不再直接调用本函数改走 wengfu_name_map 查表白名单 normalize_kio
全名形如 <中文点位名>_<位号>中文名本身不含下划线电流点位除外
其中文名会嵌入形如 PU6301_1001A 的设备号位号恒为纯 ASCII故以最后一个
ASCII 字符即中文名的结尾为锚点锚点之后的 ASCII 尾串即位号
用非 ASCII 判定而非枚举 CJK 码段可一并容纳全角符号罗马数字生僻字等
隧道窑液压顶车机B本地和远程_REM_PU5507B -> REM_PU5507B
一号干燥室侧墙温度1_TE_5113 -> TE_5113 (末尾索引数字1归名称)
一号隧道窑布煤机PU6301_1001A电流_PU6301_1001A_I -> PU6301_1001A_I
...前位限位M01_50A_PU_5503_ZAI_ZS5576 -> M01_50A_PU_5503_ZAI_ZS5576
_FLT_PU6302_2009A_22 -> FLT_PU6302_2009A_22 (无中文兜底)
切不出来时原样返回保证入库不丢数据
"""
if not name:
return name
pos = 0
for i, ch in enumerate(name):
if ord(ch) > 127: # 任意非 ASCII 字符都属于中文名
pos = i + 1
if pos == 0: # 无中文:整体即位号,去掉前导下划线
return name.lstrip('_') or name
tail = name[pos:]
if tail.startswith('_'):
return tail[1:]
m = re.match(r'^\d+_(.+)$', tail) # 名称末尾的索引数字温度1留给名称
if m:
return m.group(1)
if tail[:1].isalpha(): # 位号紧跟字母M01_50A...
return tail
return name # 兜底:无法切分则原样返回
def normalize_kio(payload):
"""KIO datachange偏移使能格式 -> [(timex, mpoint_id, value, quality)]
结构: {"PNs":{1:V,2:T,3:Q}, "PVs":{基准值}, "Objs":[{N, 1?, 2?, 3?}]}
- (1)/质量(3)缺失即等于 PVs 基准值
- 时间(2)为毫秒偏移量真实时间 = PVs 基准时间 + 偏移(ms)缺失=偏移0
- PVs 时间戳实测为北京时间(Asia/Shanghai)非手册所称 UTC
"""
obj = json.loads(payload)
pvs = obj.get('PVs', {})
base_v = pvs.get('1')
base_q = pvs.get('3')
base_ts = pvs.get('2')
base_time = None
if base_ts:
for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"):
try:
base_time = datetime.strptime(base_ts, fmt).replace(tzinfo=SHANGHAI)
break
except ValueError:
continue
records = []
for o in obj.get('Objs', []):
name = o.get('N')
if not name or base_time is None:
continue
value = o.get('1', base_v)
try:
offset_ms = int(o.get('2', 0))
except (TypeError, ValueError):
offset_ms = 0
timex = base_time + timedelta(milliseconds=offset_ms)
quality = o.get('3', base_q)
# 命中白名单则存位号,否则原样存全名(不丢、不碰撞)
mpoint_id = wengfu_name_map.get(name, name)
records.append((timex, mpoint_id, value, quality))
return records
def route_value(value):
"""按值的类型分列:返回 (val_float, val_str, val_bool, val_int)。
注意 Python bool int 子类必须先判 bool 再判 int"""
if isinstance(value, bool):
return None, None, value, None
if isinstance(value, int):
return None, None, None, value
if isinstance(value, float):
return value, None, None, None
# 字符串或其它:能转数字进 val_float否则当文本
try:
return float(value), None, None, None
except (TypeError, ValueError):
return None, (str(value) if value is not None else None), None, None
def save_items(topic, payload): def save_items(topic, payload):
table_name = TOPIC_TABLE_MAP.get(topic) table_name = TOPIC_TABLE_MAP.get(topic)
if not table_name: if not table_name:
logger.error(f"未知 topic{topic},跳过入库") logger.error(f"未知 topic{topic},跳过入库")
return return
item_list = json.loads(payload) normalize = normalize_kio if topic in KIO_TOPICS else normalize_standard
sql_str = f"INSERT INTO {table_name} (timex, mpoint_id, val_float, val_str) VALUES (:timex, :mpoint_id, :val_float, :val_str)" records = normalize(payload)
value_cols = TABLE_VALUE_COLS.get(table_name, DEFAULT_VALUE_COLS)
col_list = "timex, mpoint_id, " + ", ".join(value_cols)
param_list = ":timex, :mpoint_id, " + ", ".join(f":{c}" for c in value_cols)
sql_str = f"INSERT INTO {table_name} ({col_list}) VALUES ({param_list})"
table_mpoint_dict = mpoint_dict.setdefault(table_name, {}) table_mpoint_dict = mpoint_dict.setdefault(table_name, {})
save_list = [] save_list = []
for item in item_list: for timex, mpoint_id, value, quality in records:
timex = datetime.strptime(item['time'], "%Y%m%d%H%M%S").replace(tzinfo=tz.gettz('Asia/Shanghai'))
mpoint_id = item['name']
val_float = None
val_str = None
last_timex: datetime = table_mpoint_dict.get(mpoint_id, None) last_timex: datetime = table_mpoint_dict.get(mpoint_id, None)
if timex.minute not in getattr(conf, "SAVE_MINUTES", [2, 7, 12, 17, 22, 27, 32, 37, 42, 47, 52, 57]) or ( if timex.minute not in getattr(conf, "SAVE_MINUTES", [2, 7, 12, 17, 22, 27, 32, 37, 42, 47, 52, 57]) or (
last_timex and last_timex.minute == timex.minute): last_timex and last_timex.minute == timex.minute):
continue continue
else: else:
try: val_float, val_str, val_bool, val_int = route_value(value)
val_float = float(item["value"]) # 质量戳(Q):取不到则存 NULL
except Exception: val_q = None
val_str = item["value"] if quality is not None:
save_list.append({"timex": timex, "mpoint_id": mpoint_id, "val_float": val_float, "val_str": val_str}) try:
val_q = int(quality)
except (TypeError, ValueError):
val_q = None
save_list.append({"timex": timex, "mpoint_id": mpoint_id,
"val_float": val_float, "val_str": val_str, "val_q": val_q,
"val_bool": val_bool, "val_int": val_int})
if save_list: if save_list:
with engine.connect() as conn: with engine.connect() as conn:
@ -131,6 +292,89 @@ def save_items(topic, payload):
def get_table_stats():
"""统计 mplogx% 开头各表的数据量,返回可序列化的 dict"""
with engine.connect() as conn:
tables = conn.execute(text("""
SELECT tablename
FROM pg_tables
WHERE schemaname = current_schema()
AND tablename LIKE 'mplogx%'
ORDER BY tablename
""")).fetchall()
if not tables:
return {"tables": [], "summary": {}}
# 表名来自 pg_tables受数据库控制用双引号包裹标识符
parts = []
for (tablename,) in tables:
parts.append(f"""
SELECT
'{tablename}' AS table_name,
COUNT(DISTINCT mpoint_id)::bigint AS point_count_1h,
approximate_row_count('{tablename}')::numeric AS approximate_row_count
FROM "{tablename}"
WHERE timex >= now() - interval '1 hour'
AND timex < now()
""")
union_sql = "\nUNION ALL\n".join(parts)
rows = conn.execute(text(f"""
SELECT table_name, point_count_1h, approximate_row_count
FROM ({union_sql}) AS per_table
ORDER BY table_name
""")).fetchall()
result = []
for table_name, point_count_1h, approximate_row_count in rows:
result.append({
"table_name": table_name,
"point_count_1h": int(point_count_1h) if point_count_1h is not None else 0,
"approximate_row_count": int(approximate_row_count) if approximate_row_count is not None else 0,
})
total_points = sum(r["point_count_1h"] for r in result)
total_rows = sum(r["approximate_row_count"] for r in result)
return {
"tables": result,
"summary": {
"point_count_1h_total": total_points,
"approximate_row_count_total": total_rows,
"approximate_row_count_total_yi": round(total_rows / 100000000, 2),
},
}
class StatsHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path.split('?', 1)[0] != '/stats':
self._write_json(404, {"error": "not found"})
return
try:
self._write_json(200, get_table_stats())
except Exception as e:
logger.error("获取表统计失败", exc_info=True)
self._write_json(500, {"error": str(e)})
def _write_json(self, status, data):
body = json.dumps(data, ensure_ascii=False).encode('utf-8')
self.send_response(status)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format, *args):
logger.info("HTTP %s - %s", self.address_string(), format % args)
def start_http_server():
host = getattr(conf, 'HTTP_HOST', '0.0.0.0')
port = getattr(conf, 'HTTP_PORT', 5800)
server = ThreadingHTTPServer((host, port), StatsHandler)
logger.info(f"HTTP stats server listening on http://{host}:{port}/stats")
server.serve_forever()
def on_connect(mqttc: mqtt.Client, userdata, flags, rc, properties): def on_connect(mqttc: mqtt.Client, userdata, flags, rc, properties):
if rc == 0: if rc == 0:
logger.info("Connected to MQTT broker") logger.info("Connected to MQTT broker")
@ -153,6 +397,7 @@ def on_disconnect(mqttc: mqtt.Client, userdata, disconnect_flags, reason_code, p
def start_mqtt(): def start_mqtt():
client = mqtt.Client(callback_api_version=mqtt.CallbackAPIVersion.VERSION2, client_id='hfnf_105') client = mqtt.Client(callback_api_version=mqtt.CallbackAPIVersion.VERSION2, client_id='hfnf_105')
client.username_pw_set(conf.MQTT_USERNAME, conf.MQTT_PASSWORD)
client.on_connect = on_connect client.on_connect = on_connect
client.on_message = on_message client.on_message = on_message
client.on_disconnect = on_disconnect client.on_disconnect = on_disconnect
@ -164,9 +409,11 @@ if __name__ == '__main__':
try: try:
engine = create_engine(conf.DATABASE_URL) engine = create_engine(conf.DATABASE_URL)
logger.info("Connected to database") logger.info("Connected to database")
load_wengfu_name_map(engine) # 载入翁福位号白名单
for _topic in TOPIC_TABLE_MAP: for _topic in TOPIC_TABLE_MAP:
Thread(target=worker, args=(_topic,), daemon=True).start() Thread(target=worker, args=(_topic,), daemon=True).start()
logger.info(f"Worker thread started for topic={_topic} -> {TOPIC_TABLE_MAP[_topic]}") logger.info(f"Worker thread started for topic={_topic} -> {TOPIC_TABLE_MAP[_topic]}")
Thread(target=start_http_server, daemon=True).start()
start_mqtt() start_mqtt()
except Exception: except Exception:
logger.error("异常退出", exc_info=True) logger.error("异常退出", exc_info=True)

13
supervisor_api.conf Normal file
View File

@ -0,0 +1,13 @@
[program:mtc_api]
command=/home/zc/mtc/.venv/bin/uvicorn api:app --host 0.0.0.0 --port 5900
directory=/home/zc/mtc
user=root
autostart=true
autorestart=true
startsecs=5
startretries=3
redirect_stderr=true
stdout_logfile=/var/log/supervisor/mtc_api.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=3
environment=LANG="en_US.UTF-8",LC_ALL="en_US.UTF-8"