zcdsj
/
factory
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 报表模块基础开发

This commit is contained in:
caoqianming 2023-05-26 13:02:43 +08:00
parent a1efcb46da
commit a3d9fe4606
9 changed files with 155 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apps/bi/admin.py
View File

@ -1,12 +1,7 @@
from django.contrib import admin
from apps.bi.models import Dataset, Report
from apps.bi.models import Dataset
# Register your models here.
@admin.register(Dataset)
class DatasetAdmin(admin.ModelAdmin):
date_hierarchy = 'create_time'
@admin.register(Report)
class ReportAdmin(admin.ModelAdmin):
date_hierarchy = 'create_time'

29
apps/bi/migrations/0001_initial.py
View File

@ -1,4 +1,4 @@
# Generated by Django 3.2.12 on 2023-05-10 05:18
# Generated by Django 3.2.12 on 2023-05-24 08:06
from django.conf import settings
from django.db import migrations, models
@ -11,8 +11,8 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
('system', '0002_myschedule'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('system', '0002_myschedule'),
]
operations = [
@ -24,8 +24,10 @@ class Migration(migrations.Migration):
('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')),
('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')),
('name', models.CharField(max_length=100, verbose_name='名称')),
('description', models.TextField(default='', verbose_name='描述说明')),
('sql_query', models.TextField(default='', verbose_name='sql查询语句')),
('code', models.CharField(blank=True, default='', max_length=100, verbose_name='标识')),
('description', models.TextField(blank=True, default='', verbose_name='描述说明')),
('sql_query', models.TextField(blank=True, default='', verbose_name='sql查询语句')),
('echart_options', models.TextField(blank=True, default='')),
('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_belong_dept', to='system.dept', verbose_name='所属部门')),
('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')),
('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')),
@ -34,23 +36,4 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
migrations.CreateModel(
name='Report',
fields=[
('id', models.CharField(editable=False, help_text='主键ID', max_length=20, primary_key=True, serialize=False, verbose_name='主键ID')),
('create_time', models.DateTimeField(default=django.utils.timezone.now, help_text='创建时间', verbose_name='创建时间')),
('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')),
('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')),
('name', models.CharField(max_length=100, verbose_name='名称')),
('code', models.CharField(default='', max_length=100, verbose_name='标识')),
('js_function', models.TextField(default='', verbose_name='数据转化函数')),
('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_belong_dept', to='system.dept', verbose_name='所属部门')),
('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')),
('datasets', models.ManyToManyField(blank=True, to='bi.Dataset', verbose_name='关联数据集')),
('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')),
],
options={
'abstract': False,
},
),
]

33
apps/bi/migrations/0002_auto_20230522_1505.py
View File

@ -1,33 +0,0 @@
# Generated by Django 3.2.12 on 2023-05-22 07:05
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('bi', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='dataset',
name='description',
field=models.TextField(blank=True, default='', verbose_name='描述说明'),
),
migrations.AlterField(
model_name='dataset',
name='sql_query',
field=models.TextField(blank=True, default='', verbose_name='sql查询语句'),
),
migrations.AlterField(
model_name='report',
name='code',
field=models.CharField(blank=True, default='', max_length=100, verbose_name='标识'),
),
migrations.AlterField(
model_name='report',
name='js_function',
field=models.TextField(blank=True, default='', verbose_name='数据转化函数'),
),
]

12
apps/bi/models.py
View File

@ -4,12 +4,14 @@ from apps.utils.models import BaseModel, CommonADModel, CommonBDModel
class Dataset(CommonBDModel):
name = models.CharField('名称', max_length=100)
code = models.CharField('标识', max_length=100, default='', blank=True)
description = models.TextField('描述说明', default='', blank=True)
sql_query = models.TextField('sql查询语句', default='', blank=True)
echart_options = models.TextField(default='', blank=True)
class Report(CommonBDModel):
name = models.CharField('名称', max_length=100)
code = models.CharField('标识', max_length=100, default='', blank=True)
js_function = models.TextField('数据转化函数', default='', blank=True)
datasets = models.ManyToManyField(Dataset, verbose_name='关联数据集', blank=True)
# class Report(CommonBDModel):
# name = models.CharField('名称', max_length=100)
# code = models.CharField('标识', max_length=100, default='', blank=True)
# js_function = models.TextField('数据转化函数', default='', blank=True)
# datasets = models.ManyToManyField(Dataset, verbose_name='关联数据集', blank=True)

20
apps/bi/serializers.py
View File

@ -1,5 +1,5 @@
from apps.utils.serializers import CustomModelSerializer
from apps.bi.models import Dataset, Report
from apps.bi.models import Dataset
from apps.utils.constants import EXCLUDE_FIELDS
from rest_framework import serializers
from apps.bi.services import check_sql_safe
@ -22,17 +22,13 @@ class DatasetSerializer(CustomModelSerializer):
fields = '__all__'
class ReportCreateUpdateSerializer(CustomModelSerializer):
class Meta:
model = Report
exclude = EXCLUDE_FIELDS
# class ReportSerializer(CustomModelSerializer):
# class Meta:
# model = Report
# fields = '__all__'
# read_only_fields = EXCLUDE_FIELDS
class ReportSerializer(CustomModelSerializer):
class Meta:
model = Report
fields = '__all__'
class ReportExecSerializer(serializers.Serializer):
class DataExecSerializer(serializers.Serializer):
query = serializers.JSONField(label="查询字典参数", required=False, allow_null=True)
return_type = serializers.IntegerField(label="返回格式", required=False, default=2)

14
apps/bi/services.py
View File

@ -1,6 +1,6 @@
from apps.bi.models import Dataset, Report
from apps.utils.decorators import auto_log
from rest_framework.exceptions import ParseError
import json
from jinja2 import Template
forbidden_keywords = ["UPDATE", "DELETE", "DROP", "TRUNCATE"]
@ -13,3 +13,13 @@ def check_sql_safe(sql: str):
if kw in sql_upper:
raise ParseError('sql查询有风险')
return sql
def format_json_with_placeholders(json_str, **kwargs):
formatted_json = json_str
# 遍历关键字参数，将占位符替换为对应的值
for key, value in kwargs.items():
formatted_json = formatted_json.replace("{" + key + "}", json.dumps(value))
# 格式化后的字符串依然是 JSON 字符串，没有使用 json.loads()
return formatted_json

4
apps/bi/urls.py
View File

@ -1,13 +1,13 @@
from django.urls import path, include
from rest_framework.routers import DefaultRouter
from apps.bi.views import ReportViewSet, DatasetViewSet
from apps.bi.views import DatasetViewSet
API_BASE_URL = 'api/bi/'
HTML_BASE_URL = 'bi/'
router = DefaultRouter()
router.register('dataset', DatasetViewSet, basename='dataset')
router.register('report', ReportViewSet, basename='report')
# router.register('report', ReportViewSet, basename='report')
urlpatterns = [
path(API_BASE_URL, include(router.urls)),
]

146
apps/bi/views.py
View File

@ -2,12 +2,15 @@ from django.shortcuts import render
from apps.utils.viewsets import CustomModelViewSet
from rest_framework.decorators import action
from rest_framework.response import Response
from apps.bi.models import Dataset, Report
from apps.bi.serializers import DatasetSerializer, DatasetCreateUpdateSerializer, ReportCreateUpdateSerializer, ReportSerializer, ReportExecSerializer
from apps.bi.models import Dataset
from apps.bi.serializers import DatasetSerializer, DatasetCreateUpdateSerializer, DataExecSerializer
from django.apps import apps
from rest_framework import serializers
import concurrent.futures
from django.core.cache import cache
from apps.utils.sql import execute_raw_sql
from apps.bi.services import check_sql_safe
from apps.utils.sql import execute_raw_sql, format_sqldata
from apps.bi.services import check_sql_safe, format_json_with_placeholders
from rest_framework.exceptions import ParseError
# Create your views here.
class DatasetViewSet(CustomModelViewSet):
@ -15,54 +18,111 @@ class DatasetViewSet(CustomModelViewSet):
serializer_class = DatasetSerializer
create_serializer_class = DatasetCreateUpdateSerializer
update_serializer_class = DatasetCreateUpdateSerializer
search_fields = ['name']
class ReportViewSet(CustomModelViewSet):
queryset = Report.objects.all()
serializer_class = ReportSerializer
create_serializer_class = ReportCreateUpdateSerializer
update_serializer_class = ReportCreateUpdateSerializer
search_fields = ['name', 'code']
@action(methods=['post'], detail=True, perms_map={'post': '*'}, serializer_class=ReportExecSerializer)
@action(methods=['post'], detail=True, perms_map={'post': 'dataset.exec'}, serializer_class=DataExecSerializer, cache_seconds=0)
def exec(self, request, pk=None):
"""执行报表查询
执行报表查询并用于返回前端渲染
"""执行sql查询
"""
report = self.get_object()
rdata = ReportSerializer(instance=report).data
dt = self.get_object()
rdata = DatasetSerializer(instance=dt).data
query = request.data.get('query', {})
return_type = request.data.get('return_type', 2)
query['r_user'] = request.user.id
query['r_dept'] = request.user.belong_dept.id if request.user.belong_dept else ''
datasets = report.datasets.all()
results = {}
seconds = 10 # 缓存秒数
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: # 多线程运行并返回字典结果
fun_ps = []
for ds in datasets:
sql_query = ds.sql_query
if sql_query:
sql_f = check_sql_safe(sql_query.format(**query)) # 有风险先这样处理一下
res = cache.get(sql_f, None)
if isinstance(res, list):
results[ds.name] = res
seconds = 10
if dt.sql_query:
sql_f_ = check_sql_safe(dt.sql_query.format(**query))
sql_f_l = sql_f_.strip(';').split(';')
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: # 多线程运行并返回字典结果
fun_ps = []
for ind, val in enumerate(sql_f_l):
res = cache.get(val, None)
if isinstance(res, tuple):
results[f'ds{ind}'] = format_sqldata(res[0], res[1], return_type)
else:
fun_ps.append((ds.name, execute_raw_sql, sql_f, seconds))
# 生成执行函数
futures = {executor.submit(i[1], i[2]): i for i in fun_ps}
for future in concurrent.futures.as_completed(futures):
name, *_, sql_f, seconds = futures[future] # 获取对应的键
try:
r = future.result()
results[name] = r
if seconds:
cache.set(sql_f, r, seconds)
except Exception as e:
results[name] = 'error: ' + str(e)
fun_ps.append((f'ds{ind}', execute_raw_sql, val))
# 生成执行函数
futures = {executor.submit(i[1], i[2]): i for i in fun_ps}
for future in concurrent.futures.as_completed(futures):
name, *_, sql_f = futures[future] # 获取对应的键
try:
res = future.result()
results[name] = format_sqldata(res[0], res[1], return_type)
if seconds:
cache.set(sql_f, res, seconds)
except Exception as e:
results[name] = 'error: ' + str(e)
rdata['data'] = results
if rdata['echart_options']:
for key in results:
if isinstance(results[key], str):
raise ParseError(results[key])
rdata['echart_options'] = format_json_with_placeholders(rdata['echart_options'], **results)
return Response(rdata)
@action(methods=['get'], detail=False, perms_map={'get': '*'})
def base(self, request, pk=None):
all_models = apps.get_models()
rdict = {}
# 遍历所有模型
for model in all_models:
# 获取表名称
table_name = model._meta.db_table
rdict[table_name] = []
# 获取字段信息
fields = model._meta.get_fields()
for field in fields:
rdict[table_name].append({'name': field.name, 'type': field.get_internal_type()})
return Response(rdict)
# class ReportViewSet(CustomModelViewSet): # 暂时不用了
# queryset = Report.objects.all()
# serializer_class = ReportSerializer
# search_fields = ['name', 'code']
# @action(methods=['post'], detail=True, perms_map={'post': 'report.exec'}, serializer_class=DataExecSerializer, cache_seconds=0)
# def exec(self, request, pk=None):
# """执行报表查询
# 执行报表查询并用于返回前端渲染
# """
# report = self.get_object()
# rdata = ReportSerializer(instance=report).data
# query = request.data.get('query', {})
# return_type = request.data.get('return_type', 2)
# query['r_user'] = request.user.id
# query['r_dept'] = request.user.belong_dept.id if request.user.belong_dept else ''
# datasets = report.datasets.all()
# results = {}
# seconds = 10 # 缓存秒数
# with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: # 多线程运行并返回字典结果
# fun_ps = []
# for ds in datasets:
# sql_query = ds.sql_query
# if sql_query:
# sql_f = check_sql_safe(sql_query.format(**query)) # 有风险先这样处理一下
# res = cache.get(sql_f, None)
# if isinstance(res, tuple):
# results[ds.name] = format_sqldata(res[0], res[1], return_type)
# else:
# fun_ps.append((ds.name, execute_raw_sql, sql_f))
# # 生成执行函数
# futures = {executor.submit(i[1], i[2]): i for i in fun_ps}
# for future in concurrent.futures.as_completed(futures):
# name, *_, sql_f = futures[future] # 获取对应的键
# try:
# res = future.result()
# results[name] = format_sqldata(res[0], res[1], return_type)
# if seconds:
# cache.set(sql_f, res, seconds)
# except Exception as e:
# results[name] = 'error: ' + str(e)
# rdata['data'] = results
# return Response(rdata)

20
apps/utils/sql.py
View File

@ -1,26 +1,28 @@
from django.db import connection
def execute_raw_sql(sql: str, params=None, return_type: int =1):
"""执行原始sql并返回数据
def execute_raw_sql(sql: str, params=None):
"""执行原始sql并返回rows, columns数据
Args:
sql (str): 查询语句
params (_type_, optional): 参数列表. Defaults to None.
return_type (int, optional): 返回格式. Defaults to 1.
1 直接返回包含多个字典的列表或空列表
2 返回原始数据
"""
with connection.cursor() as cursor:
cursor.execute("SET statement_timeout TO %s;", [30000])
if params:
cursor.execute(sql, params=params)
else:
cursor.execute(sql)
columns = [desc[0] for desc in cursor.description]
rows = cursor.fetchall()
if return_type == 1:
return [dict(zip(columns, row)) for row in rows]
else:
return rows
return columns, rows
def format_sqldata(columns, rows, return_type=2):
if return_type == 2:
return [columns] + rows
elif return_type == 1:
return [dict(zip(columns, row)) for row in rows]
def query_all_dict(sql, params=None):
'''