feat: 增加报表模块

This commit is contained in:
caoqianming 2023-05-11 16:20:36 +08:00
parent 4e6f4b249b
commit b1ded623fa
14 changed files with 285 additions and 2 deletions

0
apps/bi/__init__.py Normal file
View File

12
apps/bi/admin.py Normal file
View File

@ -0,0 +1,12 @@
from django.contrib import admin
from apps.bi.models import Dataset, Report
# Register your models here.
@admin.register(Dataset)
class DatasetAdmin(admin.ModelAdmin):
date_hierarchy = 'create_time'
@admin.register(Report)
class ReportAdmin(admin.ModelAdmin):
date_hierarchy = 'create_time'

7
apps/bi/apps.py Normal file
View File

@ -0,0 +1,7 @@
from django.apps import AppConfig
class BiConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'apps.bi'
verbose_name = '报表模块'

View File

@ -0,0 +1,56 @@
# Generated by Django 3.2.12 on 2023-05-10 05:18
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
initial = True
dependencies = [
('system', '0002_myschedule'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='Dataset',
fields=[
('id', models.CharField(editable=False, help_text='主键ID', max_length=20, primary_key=True, serialize=False, verbose_name='主键ID')),
('create_time', models.DateTimeField(default=django.utils.timezone.now, help_text='创建时间', verbose_name='创建时间')),
('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')),
('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')),
('name', models.CharField(max_length=100, verbose_name='名称')),
('description', models.TextField(default='', verbose_name='描述说明')),
('sql_query', models.TextField(default='', verbose_name='sql查询语句')),
('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_belong_dept', to='system.dept', verbose_name='所属部门')),
('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')),
('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Report',
fields=[
('id', models.CharField(editable=False, help_text='主键ID', max_length=20, primary_key=True, serialize=False, verbose_name='主键ID')),
('create_time', models.DateTimeField(default=django.utils.timezone.now, help_text='创建时间', verbose_name='创建时间')),
('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')),
('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')),
('name', models.CharField(max_length=100, verbose_name='名称')),
('code', models.CharField(default='', max_length=100, verbose_name='标识')),
('js_function', models.TextField(default='', verbose_name='数据转化函数')),
('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_belong_dept', to='system.dept', verbose_name='所属部门')),
('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')),
('datasets', models.ManyToManyField(blank=True, to='bi.Dataset', verbose_name='关联数据集')),
('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')),
],
options={
'abstract': False,
},
),
]

View File

15
apps/bi/models.py Normal file
View File

@ -0,0 +1,15 @@
from django.db import models
from apps.utils.models import BaseModel, CommonADModel, CommonBDModel
# Create your models here.
class Dataset(CommonBDModel):
name = models.CharField('名称', max_length=100)
description = models.TextField('描述说明', default='', blank=True)
sql_query = models.TextField('sql查询语句', default='', blank=True)
class Report(CommonBDModel):
name = models.CharField('名称', max_length=100)
code = models.CharField('标识', max_length=100, default='', blank=True)
js_function = models.TextField('数据转化函数', default='', blank=True)
datasets = models.ManyToManyField(Dataset, verbose_name='关联数据集', blank=True)

38
apps/bi/serializers.py Normal file
View File

@ -0,0 +1,38 @@
from apps.utils.serializers import CustomModelSerializer
from apps.bi.models import Dataset, Report
from apps.utils.constants import EXCLUDE_FIELDS
from rest_framework import serializers
from apps.bi.services import check_sql_safe
class DatasetCreateUpdateSerializer(CustomModelSerializer):
class Meta:
model = Dataset
exclude = EXCLUDE_FIELDS
def validate(self, attrs):
sql_query = attrs['sql_query']
if sql_query:
check_sql_safe(sql_query)
return attrs
class DatasetSerializer(CustomModelSerializer):
class Meta:
model = Dataset
fields = '__all__'
class ReportCreateUpdateSerializer(CustomModelSerializer):
class Meta:
model = Report
exclude = EXCLUDE_FIELDS
class ReportSerializer(CustomModelSerializer):
class Meta:
model = Report
fields = '__all__'
class ReportExecSerializer(serializers.Serializer):
query = serializers.JSONField(label="查询字典参数", required=False, allow_null=True)

15
apps/bi/services.py Normal file
View File

@ -0,0 +1,15 @@
from apps.bi.models import Dataset, Report
from apps.utils.decorators import auto_log
from rest_framework.exceptions import ParseError
forbidden_keywords = ["UPDATE", "DELETE", "DROP", "TRUNCATE"]
def check_sql_safe(sql: str):
"""检查sql安全性
"""
sql_upper = sql.upper()
for kw in forbidden_keywords:
if kw in sql_upper:
raise ParseError('sql查询有风险')
return sql

3
apps/bi/tests.py Normal file
View File

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

13
apps/bi/urls.py Normal file
View File

@ -0,0 +1,13 @@
from django.urls import path, include
from rest_framework.routers import DefaultRouter
from apps.bi.views import ReportViewSet, DatasetViewSet
API_BASE_URL = 'api/bi/'
HTML_BASE_URL = 'bi/'
router = DefaultRouter()
router.register('dataset', DatasetViewSet, basename='dataset')
router.register('report', ReportViewSet, basename='report')
urlpatterns = [
path(API_BASE_URL, include(router.urls)),
]

68
apps/bi/views.py Normal file
View File

@ -0,0 +1,68 @@
from django.shortcuts import render
from apps.utils.viewsets import CustomModelViewSet
from rest_framework.decorators import action
from rest_framework.response import Response
from apps.bi.models import Dataset, Report
from apps.bi.serializers import DatasetSerializer, DatasetCreateUpdateSerializer, ReportCreateUpdateSerializer, ReportSerializer, ReportExecSerializer
import concurrent.futures
from django.core.cache import cache
from apps.utils.sql import execute_raw_sql
from apps.bi.services import check_sql_safe
# Create your views here.
class DatasetViewSet(CustomModelViewSet):
queryset = Dataset.objects.all()
serializer_class = DatasetSerializer
create_serializer_class = DatasetCreateUpdateSerializer
update_serializer_class = DatasetCreateUpdateSerializer
search_fields = ['name']
class ReportViewSet(CustomModelViewSet):
queryset = Report.objects.all()
serializer_class = ReportSerializer
create_serializer_class = ReportCreateUpdateSerializer
update_serializer_class = ReportCreateUpdateSerializer
search_fields = ['name', 'code']
@action(methods=['post'], detail=True, perms_map={'post': '*'}, serializer_class=ReportExecSerializer)
def exec(self, request, pk=None):
"""执行报表查询
执行报表查询并用于返回前端渲染
"""
report = self.get_object()
rdata = ReportSerializer(instance=report).data
query = request.data.get('query', {})
query['r_user'] = request.user.id
query['r_dept'] = request.user.belong_dept.id if request.user.belong_dept else ''
datasets = report.datasets.all()
results = {}
seconds = 10 # 缓存秒数
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: # 多线程运行并返回字典结果
fun_ps = []
for ds in datasets:
sql_query = ds.sql_query
if sql_query:
sql_f = check_sql_safe(sql_query.format(**query)) # 有风险先这样处理一下
res = cache.get(sql_f, None)
if isinstance(res, list):
results[ds.name] = res
else:
fun_ps.append((ds.name, execute_raw_sql, sql_f, seconds))
# 生成执行函数
futures = {executor.submit(i[1], i[2]): i for i in fun_ps}
for future in concurrent.futures.as_completed(futures):
name, *_, sql_f, seconds = futures[future] # 获取对应的键
try:
r = future.result()
results[name] = r
if seconds:
cache.set(sql_f, r, seconds)
except Exception as e:
results[name] = 'error: ' + str(e)
rdata['data'] = results
return Response(rdata)

54
apps/utils/sql.py Normal file
View File

@ -0,0 +1,54 @@
from django.db import connection
def execute_raw_sql(sql: str, params=None, return_type: int =1):
"""执行原始sql并返回数据
Args:
sql (str): 查询语句
params (_type_, optional): 参数列表. Defaults to None.
return_type (int, optional): 返回格式. Defaults to 1.
1 直接返回包含多个字典的列表或空列表
2 返回原始数据
"""
with connection.cursor() as cursor:
if params:
cursor.execute(sql, params=params)
else:
cursor.execute(sql)
columns = [desc[0] for desc in cursor.description]
rows = cursor.fetchall()
if return_type == 1:
return [dict(zip(columns, row)) for row in rows]
else:
return rows
def query_all_dict(sql, params=None):
'''
查询所有结果返回字典类型数据
:param sql:
:param params:
:return:
'''
with connection.cursor() as cursor:
if params:
cursor.execute(sql, params=params)
else:
cursor.execute(sql)
columns = [desc[0] for desc in cursor.description]
return [dict(zip(columns, row)) for row in cursor.fetchall()]
def query_one_dict(sql, params=None):
"""
查询一个结果返回字典类型数据
:param sql:
:param params:
:return:
"""
with connection.cursor() as cursor:
if params:
cursor.execute(sql, params=params)
else:
cursor.execute(sql)
columns = [desc[0] for desc in cursor.description]
row = cursor.fetchone()
return dict(zip(columns, row))

View File

@ -29,7 +29,7 @@ DEBUG = conf.DEBUG
ALLOWED_HOSTS = ['*']
SYS_NAME = 'XT_EHS'
SYS_VERSION = '2.00.13'
SYS_VERSION = '2.01.13'
PROJECT_NAME = conf.PROJECT_NAME
@ -62,7 +62,8 @@ INSTALLED_APPS = [
'apps.am',
'apps.vm',
'apps.rpm',
'apps.opm'
'apps.opm',
'apps.bi'
]
MIDDLEWARE = [

View File

@ -54,6 +54,7 @@ urlpatterns = [
path('', include('apps.rpm.urls')),
path('', include('apps.opm.urls')),
path('', include('apps.vm.urls')),
path('', include('apps.bi.urls')),