diff --git a/apps/bi/admin.py b/apps/bi/admin.py index f345415d..03822684 100644 --- a/apps/bi/admin.py +++ b/apps/bi/admin.py @@ -1,12 +1,7 @@ from django.contrib import admin -from apps.bi.models import Dataset, Report +from apps.bi.models import Dataset # Register your models here. @admin.register(Dataset) class DatasetAdmin(admin.ModelAdmin): - date_hierarchy = 'create_time' - - -@admin.register(Report) -class ReportAdmin(admin.ModelAdmin): date_hierarchy = 'create_time' \ No newline at end of file diff --git a/apps/bi/migrations/0001_initial.py b/apps/bi/migrations/0001_initial.py index 79a13b2c..2d2c3c19 100644 --- a/apps/bi/migrations/0001_initial.py +++ b/apps/bi/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.12 on 2023-05-10 05:18 +# Generated by Django 3.2.12 on 2023-05-24 08:06 from django.conf import settings from django.db import migrations, models @@ -11,8 +11,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('system', '0002_myschedule'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('system', '0002_myschedule'), ] operations = [ @@ -24,8 +24,10 @@ class Migration(migrations.Migration): ('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')), ('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')), ('name', models.CharField(max_length=100, verbose_name='名称')), - ('description', models.TextField(default='', verbose_name='描述说明')), - ('sql_query', models.TextField(default='', verbose_name='sql查询语句')), + ('code', models.CharField(blank=True, default='', max_length=100, verbose_name='标识')), + ('description', models.TextField(blank=True, default='', verbose_name='描述说明')), + ('sql_query', models.TextField(blank=True, default='', verbose_name='sql查询语句')), + ('echart_options', models.TextField(blank=True, default='')), ('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_belong_dept', to='system.dept', verbose_name='所属部门')), ('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')), ('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')), @@ -34,23 +36,4 @@ class Migration(migrations.Migration): 'abstract': False, }, ), - migrations.CreateModel( - name='Report', - fields=[ - ('id', models.CharField(editable=False, help_text='主键ID', max_length=20, primary_key=True, serialize=False, verbose_name='主键ID')), - ('create_time', models.DateTimeField(default=django.utils.timezone.now, help_text='创建时间', verbose_name='创建时间')), - ('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')), - ('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')), - ('name', models.CharField(max_length=100, verbose_name='名称')), - ('code', models.CharField(default='', max_length=100, verbose_name='标识')), - ('js_function', models.TextField(default='', verbose_name='数据转化函数')), - ('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_belong_dept', to='system.dept', verbose_name='所属部门')), - ('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')), - ('datasets', models.ManyToManyField(blank=True, to='bi.Dataset', verbose_name='关联数据集')), - ('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')), - ], - options={ - 'abstract': False, - }, - ), ] diff --git a/apps/bi/migrations/0002_auto_20230522_1505.py b/apps/bi/migrations/0002_auto_20230522_1505.py deleted file mode 100644 index 5bee1c41..00000000 --- a/apps/bi/migrations/0002_auto_20230522_1505.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 3.2.12 on 2023-05-22 07:05 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('bi', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='dataset', - name='description', - field=models.TextField(blank=True, default='', verbose_name='描述说明'), - ), - migrations.AlterField( - model_name='dataset', - name='sql_query', - field=models.TextField(blank=True, default='', verbose_name='sql查询语句'), - ), - migrations.AlterField( - model_name='report', - name='code', - field=models.CharField(blank=True, default='', max_length=100, verbose_name='标识'), - ), - migrations.AlterField( - model_name='report', - name='js_function', - field=models.TextField(blank=True, default='', verbose_name='数据转化函数'), - ), - ] diff --git a/apps/bi/models.py b/apps/bi/models.py index bc28f5c6..b54a4b8c 100644 --- a/apps/bi/models.py +++ b/apps/bi/models.py @@ -3,13 +3,15 @@ from apps.utils.models import BaseModel, CommonADModel, CommonBDModel # Create your models here. class Dataset(CommonBDModel): - name = models.CharField('名称', max_length=100) - description = models.TextField('描述说明', default='', blank=True) - sql_query = models.TextField('sql查询语句', default='', blank=True) - - -class Report(CommonBDModel): name = models.CharField('名称', max_length=100) code = models.CharField('标识', max_length=100, default='', blank=True) - js_function = models.TextField('数据转化函数', default='', blank=True) - datasets = models.ManyToManyField(Dataset, verbose_name='关联数据集', blank=True) \ No newline at end of file + description = models.TextField('描述说明', default='', blank=True) + sql_query = models.TextField('sql查询语句', default='', blank=True) + echart_options = models.TextField(default='', blank=True) + + +# class Report(CommonBDModel): +# name = models.CharField('名称', max_length=100) +# code = models.CharField('标识', max_length=100, default='', blank=True) +# js_function = models.TextField('数据转化函数', default='', blank=True) +# datasets = models.ManyToManyField(Dataset, verbose_name='关联数据集', blank=True) \ No newline at end of file diff --git a/apps/bi/serializers.py b/apps/bi/serializers.py index c4a0c13b..b70de58c 100644 --- a/apps/bi/serializers.py +++ b/apps/bi/serializers.py @@ -1,5 +1,5 @@ from apps.utils.serializers import CustomModelSerializer -from apps.bi.models import Dataset, Report +from apps.bi.models import Dataset from apps.utils.constants import EXCLUDE_FIELDS from rest_framework import serializers from apps.bi.services import check_sql_safe @@ -22,17 +22,13 @@ class DatasetSerializer(CustomModelSerializer): fields = '__all__' -class ReportCreateUpdateSerializer(CustomModelSerializer): - class Meta: - model = Report - exclude = EXCLUDE_FIELDS +# class ReportSerializer(CustomModelSerializer): +# class Meta: +# model = Report +# fields = '__all__' +# read_only_fields = EXCLUDE_FIELDS -class ReportSerializer(CustomModelSerializer): - class Meta: - model = Report - fields = '__all__' - - -class ReportExecSerializer(serializers.Serializer): - query = serializers.JSONField(label="查询字典参数", required=False, allow_null=True) \ No newline at end of file +class DataExecSerializer(serializers.Serializer): + query = serializers.JSONField(label="查询字典参数", required=False, allow_null=True) + return_type = serializers.IntegerField(label="返回格式", required=False, default=2) \ No newline at end of file diff --git a/apps/bi/services.py b/apps/bi/services.py index 15cffdde..b6fa0e46 100644 --- a/apps/bi/services.py +++ b/apps/bi/services.py @@ -1,6 +1,6 @@ -from apps.bi.models import Dataset, Report -from apps.utils.decorators import auto_log from rest_framework.exceptions import ParseError +import json +from jinja2 import Template forbidden_keywords = ["UPDATE", "DELETE", "DROP", "TRUNCATE"] @@ -12,4 +12,14 @@ def check_sql_safe(sql: str): for kw in forbidden_keywords: if kw in sql_upper: raise ParseError('sql查询有风险') - return sql \ No newline at end of file + return sql + +def format_json_with_placeholders(json_str, **kwargs): + formatted_json = json_str + + # 遍历关键字参数，将占位符替换为对应的值 + for key, value in kwargs.items(): + formatted_json = formatted_json.replace("{" + key + "}", json.dumps(value)) + + # 格式化后的字符串依然是 JSON 字符串，没有使用 json.loads() + return formatted_json \ No newline at end of file diff --git a/apps/bi/urls.py b/apps/bi/urls.py index 4e2f409f..b349d82c 100644 --- a/apps/bi/urls.py +++ b/apps/bi/urls.py @@ -1,13 +1,13 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from apps.bi.views import ReportViewSet, DatasetViewSet +from apps.bi.views import DatasetViewSet API_BASE_URL = 'api/bi/' HTML_BASE_URL = 'bi/' router = DefaultRouter() router.register('dataset', DatasetViewSet, basename='dataset') -router.register('report', ReportViewSet, basename='report') +# router.register('report', ReportViewSet, basename='report') urlpatterns = [ path(API_BASE_URL, include(router.urls)), ] \ No newline at end of file diff --git a/apps/bi/views.py b/apps/bi/views.py index 153761dc..8e6b76ab 100644 --- a/apps/bi/views.py +++ b/apps/bi/views.py @@ -2,12 +2,15 @@ from django.shortcuts import render from apps.utils.viewsets import CustomModelViewSet from rest_framework.decorators import action from rest_framework.response import Response -from apps.bi.models import Dataset, Report -from apps.bi.serializers import DatasetSerializer, DatasetCreateUpdateSerializer, ReportCreateUpdateSerializer, ReportSerializer, ReportExecSerializer +from apps.bi.models import Dataset +from apps.bi.serializers import DatasetSerializer, DatasetCreateUpdateSerializer, DataExecSerializer +from django.apps import apps +from rest_framework import serializers import concurrent.futures from django.core.cache import cache -from apps.utils.sql import execute_raw_sql -from apps.bi.services import check_sql_safe +from apps.utils.sql import execute_raw_sql, format_sqldata +from apps.bi.services import check_sql_safe, format_json_with_placeholders +from rest_framework.exceptions import ParseError # Create your views here. class DatasetViewSet(CustomModelViewSet): @@ -15,54 +18,111 @@ class DatasetViewSet(CustomModelViewSet): serializer_class = DatasetSerializer create_serializer_class = DatasetCreateUpdateSerializer update_serializer_class = DatasetCreateUpdateSerializer - search_fields = ['name'] - - -class ReportViewSet(CustomModelViewSet): - queryset = Report.objects.all() - serializer_class = ReportSerializer - create_serializer_class = ReportCreateUpdateSerializer - update_serializer_class = ReportCreateUpdateSerializer search_fields = ['name', 'code'] - @action(methods=['post'], detail=True, perms_map={'post': '*'}, serializer_class=ReportExecSerializer) + @action(methods=['post'], detail=True, perms_map={'post': 'dataset.exec'}, serializer_class=DataExecSerializer, cache_seconds=0) def exec(self, request, pk=None): - """执行报表查询 - - 执行报表查询并用于返回前端渲染 + """执行sql查询 """ - report = self.get_object() - rdata = ReportSerializer(instance=report).data + dt = self.get_object() + rdata = DatasetSerializer(instance=dt).data query = request.data.get('query', {}) + return_type = request.data.get('return_type', 2) query['r_user'] = request.user.id query['r_dept'] = request.user.belong_dept.id if request.user.belong_dept else '' - datasets = report.datasets.all() results = {} - seconds = 10 # 缓存秒数 - - with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: # 多线程运行并返回字典结果 - fun_ps = [] - for ds in datasets: - sql_query = ds.sql_query - if sql_query: - sql_f = check_sql_safe(sql_query.format(**query)) # 有风险先这样处理一下 - res = cache.get(sql_f, None) - if isinstance(res, list): - results[ds.name] = res + seconds = 10 + if dt.sql_query: + sql_f_ = check_sql_safe(dt.sql_query.format(**query)) + sql_f_l = sql_f_.strip(';').split(';') + with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: # 多线程运行并返回字典结果 + fun_ps = [] + for ind, val in enumerate(sql_f_l): + res = cache.get(val, None) + if isinstance(res, tuple): + results[f'ds{ind}'] = format_sqldata(res[0], res[1], return_type) else: - fun_ps.append((ds.name, execute_raw_sql, sql_f, seconds)) - # 生成执行函数 - futures = {executor.submit(i[1], i[2]): i for i in fun_ps} - for future in concurrent.futures.as_completed(futures): - name, *_, sql_f, seconds = futures[future] # 获取对应的键 - try: - r = future.result() - results[name] = r - if seconds: - cache.set(sql_f, r, seconds) - except Exception as e: - results[name] = 'error: ' + str(e) - + fun_ps.append((f'ds{ind}', execute_raw_sql, val)) + # 生成执行函数 + futures = {executor.submit(i[1], i[2]): i for i in fun_ps} + for future in concurrent.futures.as_completed(futures): + name, *_, sql_f = futures[future] # 获取对应的键 + try: + res = future.result() + results[name] = format_sqldata(res[0], res[1], return_type) + if seconds: + cache.set(sql_f, res, seconds) + except Exception as e: + results[name] = 'error: ' + str(e) rdata['data'] = results + if rdata['echart_options']: + for key in results: + if isinstance(results[key], str): + raise ParseError(results[key]) + rdata['echart_options'] = format_json_with_placeholders(rdata['echart_options'], **results) return Response(rdata) + + @action(methods=['get'], detail=False, perms_map={'get': '*'}) + def base(self, request, pk=None): + all_models = apps.get_models() + rdict = {} + # 遍历所有模型 + for model in all_models: + # 获取表名称 + table_name = model._meta.db_table + rdict[table_name] = [] + + # 获取字段信息 + fields = model._meta.get_fields() + for field in fields: + rdict[table_name].append({'name': field.name, 'type': field.get_internal_type()}) + return Response(rdict) + + +# class ReportViewSet(CustomModelViewSet): # 暂时不用了 +# queryset = Report.objects.all() +# serializer_class = ReportSerializer +# search_fields = ['name', 'code'] + +# @action(methods=['post'], detail=True, perms_map={'post': 'report.exec'}, serializer_class=DataExecSerializer, cache_seconds=0) +# def exec(self, request, pk=None): +# """执行报表查询 + +# 执行报表查询并用于返回前端渲染 +# """ +# report = self.get_object() +# rdata = ReportSerializer(instance=report).data +# query = request.data.get('query', {}) +# return_type = request.data.get('return_type', 2) +# query['r_user'] = request.user.id +# query['r_dept'] = request.user.belong_dept.id if request.user.belong_dept else '' +# datasets = report.datasets.all() +# results = {} +# seconds = 10 # 缓存秒数 + +# with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: # 多线程运行并返回字典结果 +# fun_ps = [] +# for ds in datasets: +# sql_query = ds.sql_query +# if sql_query: +# sql_f = check_sql_safe(sql_query.format(**query)) # 有风险先这样处理一下 +# res = cache.get(sql_f, None) +# if isinstance(res, tuple): +# results[ds.name] = format_sqldata(res[0], res[1], return_type) +# else: +# fun_ps.append((ds.name, execute_raw_sql, sql_f)) +# # 生成执行函数 +# futures = {executor.submit(i[1], i[2]): i for i in fun_ps} +# for future in concurrent.futures.as_completed(futures): +# name, *_, sql_f = futures[future] # 获取对应的键 +# try: +# res = future.result() +# results[name] = format_sqldata(res[0], res[1], return_type) +# if seconds: +# cache.set(sql_f, res, seconds) +# except Exception as e: +# results[name] = 'error: ' + str(e) + +# rdata['data'] = results +# return Response(rdata) \ No newline at end of file diff --git a/apps/utils/sql.py b/apps/utils/sql.py index 3262282b..f90de8a5 100644 --- a/apps/utils/sql.py +++ b/apps/utils/sql.py @@ -1,26 +1,28 @@ from django.db import connection -def execute_raw_sql(sql: str, params=None, return_type: int =1): - """执行原始sql并返回数据 +def execute_raw_sql(sql: str, params=None): + """执行原始sql并返回rows, columns数据 Args: sql (str): 查询语句 params (_type_, optional): 参数列表. Defaults to None. - return_type (int, optional): 返回格式. Defaults to 1. - 1 直接返回包含多个字典的列表或空列表 - 2 返回原始数据 """ with connection.cursor() as cursor: + cursor.execute("SET statement_timeout TO %s;", [30000]) if params: cursor.execute(sql, params=params) else: cursor.execute(sql) columns = [desc[0] for desc in cursor.description] rows = cursor.fetchall() - if return_type == 1: - return [dict(zip(columns, row)) for row in rows] - else: - return rows + return columns, rows + +def format_sqldata(columns, rows, return_type=2): + if return_type == 2: + return [columns] + rows + elif return_type == 1: + return [dict(zip(columns, row)) for row in rows] + def query_all_dict(sql, params=None): '''