feat: 报表模块基础开发

apps/bi/admin.py

@@ -1,12 +1,7 @@
 from django.contrib import admin
-from apps.bi.models import Dataset, Report
+from apps.bi.models import Dataset
 # Register your models here.
 
 @admin.register(Dataset)
 class DatasetAdmin(admin.ModelAdmin):
-    date_hierarchy = 'create_time'
-
-
-@admin.register(Report)
-class ReportAdmin(admin.ModelAdmin):
     date_hierarchy = 'create_time'

apps/bi/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.2.12 on 2023-05-10 05:18
+# Generated by Django 3.2.12 on 2023-05-24 08:06
 
 from django.conf import settings
 from django.db import migrations, models
@@ -11,8 +11,8 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('system', '0002_myschedule'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('system', '0002_myschedule'),
     ]
 
     operations = [
@@ -24,8 +24,10 @@ class Migration(migrations.Migration):
                 ('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')),
                 ('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')),
                 ('name', models.CharField(max_length=100, verbose_name='名称')),
-                ('description', models.TextField(default='', verbose_name='描述说明')),
+                ('code', models.CharField(blank=True, default='', max_length=100, verbose_name='标识')),
-                ('sql_query', models.TextField(default='', verbose_name='sql查询语句')),
+                ('description', models.TextField(blank=True, default='', verbose_name='描述说明')),
+                ('sql_query', models.TextField(blank=True, default='', verbose_name='sql查询语句')),
+                ('echart_options', models.TextField(blank=True, default='')),
                 ('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_belong_dept', to='system.dept', verbose_name='所属部门')),
                 ('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')),
                 ('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')),
@@ -34,23 +36,4 @@ class Migration(migrations.Migration):
                 'abstract': False,
             },
         ),
-        migrations.CreateModel(
-            name='Report',
-            fields=[
-                ('id', models.CharField(editable=False, help_text='主键ID', max_length=20, primary_key=True, serialize=False, verbose_name='主键ID')),
-                ('create_time', models.DateTimeField(default=django.utils.timezone.now, help_text='创建时间', verbose_name='创建时间')),
-                ('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')),
-                ('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')),
-                ('name', models.CharField(max_length=100, verbose_name='名称')),
-                ('code', models.CharField(default='', max_length=100, verbose_name='标识')),
-                ('js_function', models.TextField(default='', verbose_name='数据转化函数')),
-                ('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_belong_dept', to='system.dept', verbose_name='所属部门')),
-                ('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')),
-                ('datasets', models.ManyToManyField(blank=True, to='bi.Dataset', verbose_name='关联数据集')),
-                ('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')),
-            ],
-            options={
-                'abstract': False,
-            },
-        ),
     ]

apps/bi/migrations/0002_auto_20230522_1505.py

@@ -1,33 +0,0 @@
-# Generated by Django 3.2.12 on 2023-05-22 07:05
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('bi', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='dataset',
-            name='description',
-            field=models.TextField(blank=True, default='', verbose_name='描述说明'),
-        ),
-        migrations.AlterField(
-            model_name='dataset',
-            name='sql_query',
-            field=models.TextField(blank=True, default='', verbose_name='sql查询语句'),
-        ),
-        migrations.AlterField(
-            model_name='report',
-            name='code',
-            field=models.CharField(blank=True, default='', max_length=100, verbose_name='标识'),
-        ),
-        migrations.AlterField(
-            model_name='report',
-            name='js_function',
-            field=models.TextField(blank=True, default='', verbose_name='数据转化函数'),
-        ),
-    ]

apps/bi/models.py

@@ -3,13 +3,15 @@ from apps.utils.models import BaseModel, CommonADModel, CommonBDModel
 # Create your models here.
 
 class Dataset(CommonBDModel):
-    name = models.CharField('名称', max_length=100)
-    description = models.TextField('描述说明', default='', blank=True)
-    sql_query = models.TextField('sql查询语句', default='', blank=True) 
-
-
-class Report(CommonBDModel):
     name = models.CharField('名称', max_length=100)
     code = models.CharField('标识', max_length=100, default='', blank=True)
-    js_function = models.TextField('数据转化函数', default='', blank=True)
+    description = models.TextField('描述说明', default='', blank=True)
-    datasets = models.ManyToManyField(Dataset, verbose_name='关联数据集', blank=True)
+    sql_query = models.TextField('sql查询语句', default='', blank=True)
+    echart_options  = models.TextField(default='', blank=True)
+
+
+# class Report(CommonBDModel):
+#     name = models.CharField('名称', max_length=100)
+#     code = models.CharField('标识', max_length=100, default='', blank=True)
+#     js_function = models.TextField('数据转化函数', default='', blank=True)
+#     datasets = models.ManyToManyField(Dataset, verbose_name='关联数据集', blank=True)

apps/bi/serializers.py

@@ -1,5 +1,5 @@
 from apps.utils.serializers import CustomModelSerializer
-from apps.bi.models import Dataset, Report
+from apps.bi.models import Dataset
 from apps.utils.constants import EXCLUDE_FIELDS
 from rest_framework import serializers
 from apps.bi.services import check_sql_safe
@@ -22,17 +22,13 @@ class DatasetSerializer(CustomModelSerializer):
         fields = '__all__'
 
 
-class ReportCreateUpdateSerializer(CustomModelSerializer):
+# class ReportSerializer(CustomModelSerializer):
-    class Meta:
+#     class Meta:
-        model = Report
+#         model = Report
-        exclude = EXCLUDE_FIELDS
+#         fields = '__all__'
+#         read_only_fields = EXCLUDE_FIELDS
 
 
-class ReportSerializer(CustomModelSerializer):
+class DataExecSerializer(serializers.Serializer):
-    class Meta:
+    query = serializers.JSONField(label="查询字典参数", required=False, allow_null=True)
-        model = Report
+    return_type = serializers.IntegerField(label="返回格式", required=False, default=2)
-        fields = '__all__'
-
-
-class ReportExecSerializer(serializers.Serializer):
-    query = serializers.JSONField(label="查询字典参数", required=False, allow_null=True)

apps/bi/services.py

@@ -1,6 +1,6 @@
-from apps.bi.models import Dataset, Report
-from apps.utils.decorators import auto_log
 from rest_framework.exceptions import ParseError
+import json
+from jinja2 import Template
 
 forbidden_keywords = ["UPDATE", "DELETE", "DROP", "TRUNCATE"]
 
@@ -12,4 +12,14 @@ def check_sql_safe(sql: str):
     for kw in forbidden_keywords:
         if kw in sql_upper:
             raise ParseError('sql查询有风险')
-    return sql
+    return sql
+
+def format_json_with_placeholders(json_str, **kwargs):
+    formatted_json = json_str
+
+    # 遍历关键字参数，将占位符替换为对应的值
+    for key, value in kwargs.items():
+        formatted_json = formatted_json.replace("{" + key + "}", json.dumps(value))
+
+    # 格式化后的字符串依然是 JSON 字符串，没有使用 json.loads()
+    return formatted_json

apps/bi/urls.py

@@ -1,13 +1,13 @@
 from django.urls import path, include
 from rest_framework.routers import DefaultRouter
-from apps.bi.views import ReportViewSet, DatasetViewSet
+from apps.bi.views import DatasetViewSet
 
 API_BASE_URL = 'api/bi/'
 HTML_BASE_URL = 'bi/'
 
 router = DefaultRouter()
 router.register('dataset', DatasetViewSet, basename='dataset')
-router.register('report', ReportViewSet, basename='report')
+# router.register('report', ReportViewSet, basename='report')
 urlpatterns = [
     path(API_BASE_URL, include(router.urls)),
 ]

apps/bi/views.py

@@ -2,12 +2,15 @@ from django.shortcuts import render
 from apps.utils.viewsets import CustomModelViewSet
 from rest_framework.decorators import action
 from rest_framework.response import Response
-from apps.bi.models import Dataset, Report
+from apps.bi.models import Dataset
-from apps.bi.serializers import DatasetSerializer, DatasetCreateUpdateSerializer, ReportCreateUpdateSerializer, ReportSerializer, ReportExecSerializer
+from apps.bi.serializers import DatasetSerializer, DatasetCreateUpdateSerializer, DataExecSerializer
+from django.apps import apps
+from rest_framework import serializers
 import concurrent.futures
 from django.core.cache import cache
-from apps.utils.sql import execute_raw_sql
+from apps.utils.sql import execute_raw_sql, format_sqldata
-from apps.bi.services import check_sql_safe
+from apps.bi.services import check_sql_safe, format_json_with_placeholders
+from rest_framework.exceptions import ParseError
 # Create your views here.
 
 class DatasetViewSet(CustomModelViewSet):
@@ -15,54 +18,111 @@ class DatasetViewSet(CustomModelViewSet):
     serializer_class = DatasetSerializer
     create_serializer_class = DatasetCreateUpdateSerializer
     update_serializer_class = DatasetCreateUpdateSerializer
-    search_fields = ['name']
-
-
-class ReportViewSet(CustomModelViewSet):
-    queryset = Report.objects.all()
-    serializer_class = ReportSerializer
-    create_serializer_class = ReportCreateUpdateSerializer
-    update_serializer_class = ReportCreateUpdateSerializer
     search_fields = ['name', 'code']
 
-    @action(methods=['post'], detail=True, perms_map={'post': '*'}, serializer_class=ReportExecSerializer)
+    @action(methods=['post'], detail=True, perms_map={'post': 'dataset.exec'}, serializer_class=DataExecSerializer, cache_seconds=0)
     def exec(self, request, pk=None):
-        """执行报表查询
+        """执行sql查询
-
-        执行报表查询并用于返回前端渲染
         """
-        report = self.get_object()
+        dt = self.get_object()
-        rdata = ReportSerializer(instance=report).data
+        rdata = DatasetSerializer(instance=dt).data
         query = request.data.get('query', {})
+        return_type = request.data.get('return_type', 2)
         query['r_user'] = request.user.id
         query['r_dept'] = request.user.belong_dept.id if request.user.belong_dept else ''
-        datasets = report.datasets.all()
         results = {}
-        seconds = 10   # 缓存秒数
+        seconds = 10
-
+        if dt.sql_query:
-        with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:   # 多线程运行并返回字典结果
+            sql_f_ = check_sql_safe(dt.sql_query.format(**query))
-            fun_ps = []
+            sql_f_l = sql_f_.strip(';').split(';')
-            for ds in datasets:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:   # 多线程运行并返回字典结果
-                sql_query = ds.sql_query
+                fun_ps = []
-                if sql_query:
+                for ind, val in  enumerate(sql_f_l):
-                    sql_f = check_sql_safe(sql_query.format(**query))  # 有风险先这样处理一下
+                    res = cache.get(val, None)
-                    res = cache.get(sql_f, None)
+                    if isinstance(res, tuple):
-                    if isinstance(res, list):
+                        results[f'ds{ind}'] = format_sqldata(res[0], res[1], return_type)
-                        results[ds.name] = res
                     else:
-                        fun_ps.append((ds.name, execute_raw_sql, sql_f, seconds))
+                        fun_ps.append((f'ds{ind}', execute_raw_sql, val))
-            # 生成执行函数
+                # 生成执行函数
-            futures = {executor.submit(i[1], i[2]): i for i in fun_ps}
+                futures = {executor.submit(i[1], i[2]): i for i in fun_ps}
-            for future in concurrent.futures.as_completed(futures):
+                for future in concurrent.futures.as_completed(futures):
-                name, *_, sql_f, seconds = futures[future]  # 获取对应的键
+                    name, *_, sql_f = futures[future]  # 获取对应的键
-                try:
+                    try:
-                    r = future.result()
+                        res = future.result()
-                    results[name] = r
+                        results[name] = format_sqldata(res[0], res[1], return_type)
-                    if seconds:
+                        if seconds:
-                        cache.set(sql_f, r, seconds)
+                            cache.set(sql_f, res, seconds)
-                except Exception as e:
+                    except Exception as e:
-                    results[name] = 'error: ' + str(e)
+                        results[name] = 'error: ' + str(e)
-
         rdata['data'] = results
+        if rdata['echart_options']:
+            for key in results:
+                if isinstance(results[key], str):
+                    raise ParseError(results[key])
+            rdata['echart_options'] = format_json_with_placeholders(rdata['echart_options'], **results)
         return Response(rdata)
+                
+    @action(methods=['get'], detail=False, perms_map={'get': '*'})
+    def base(self, request, pk=None):
+        all_models = apps.get_models()
+        rdict = {}
+        # 遍历所有模型
+        for model in all_models:
+            # 获取表名称
+            table_name = model._meta.db_table
+            rdict[table_name] = []
+
+            # 获取字段信息
+            fields = model._meta.get_fields()
+            for field in fields:
+                rdict[table_name].append({'name': field.name, 'type': field.get_internal_type()})
+        return Response(rdict)
+
+
+# class ReportViewSet(CustomModelViewSet):   # 暂时不用了
+#     queryset = Report.objects.all()
+#     serializer_class = ReportSerializer
+#     search_fields = ['name', 'code']
+
+#     @action(methods=['post'], detail=True, perms_map={'post': 'report.exec'}, serializer_class=DataExecSerializer, cache_seconds=0)
+#     def exec(self, request, pk=None):
+#         """执行报表查询
+
+#         执行报表查询并用于返回前端渲染
+#         """
+#         report = self.get_object()
+#         rdata = ReportSerializer(instance=report).data
+#         query = request.data.get('query', {})
+#         return_type = request.data.get('return_type', 2)
+#         query['r_user'] = request.user.id
+#         query['r_dept'] = request.user.belong_dept.id if request.user.belong_dept else ''
+#         datasets = report.datasets.all()
+#         results = {}
+#         seconds = 10   # 缓存秒数
+
+#         with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:   # 多线程运行并返回字典结果
+#             fun_ps = []
+#             for ds in datasets:
+#                 sql_query = ds.sql_query
+#                 if sql_query:
+#                     sql_f = check_sql_safe(sql_query.format(**query))  # 有风险先这样处理一下
+#                     res = cache.get(sql_f, None)
+#                     if isinstance(res, tuple):
+#                         results[ds.name] = format_sqldata(res[0], res[1], return_type)
+#                     else:
+#                         fun_ps.append((ds.name, execute_raw_sql, sql_f))
+#             # 生成执行函数
+#             futures = {executor.submit(i[1], i[2]): i for i in fun_ps}
+#             for future in concurrent.futures.as_completed(futures):
+#                 name, *_, sql_f = futures[future]  # 获取对应的键
+#                 try:
+#                     res = future.result()
+#                     results[name] = format_sqldata(res[0], res[1], return_type)
+#                     if seconds:
+#                         cache.set(sql_f, res, seconds)
+#                 except Exception as e:
+#                     results[name] = 'error: ' + str(e)
+
+#         rdata['data'] = results
+#         return Response(rdata)
     

apps/utils/sql.py

@@ -1,26 +1,28 @@
 from django.db import connection
 
-def execute_raw_sql(sql: str, params=None, return_type: int =1):
+def execute_raw_sql(sql: str, params=None):
-    """执行原始sql并返回数据
+    """执行原始sql并返回rows, columns数据
 
     Args:
         sql (str): 查询语句
         params (_type_, optional): 参数列表. Defaults to None.
-        return_type (int, optional): 返回格式. Defaults to 1.
-        1 直接返回包含多个字典的列表或空列表
-        2 返回原始数据
     """
     with connection.cursor() as cursor:
+        cursor.execute("SET statement_timeout TO %s;", [30000])
         if params:
             cursor.execute(sql, params=params)
         else:
             cursor.execute(sql)
         columns  = [desc[0] for desc in cursor.description]
         rows = cursor.fetchall()
-        if return_type == 1:
+        return columns, rows
-            return [dict(zip(columns, row)) for row in rows]
+    
-        else:
+def format_sqldata(columns, rows, return_type=2):
-            return rows
+    if return_type == 2:
+        return [columns] + rows
+    elif return_type == 1:
+        return [dict(zip(columns, row)) for row in rows]
+
 
 def query_all_dict(sql, params=None):
     '''