From b1ded623fa5dfaad2a974b31adc98b9575d9255c Mon Sep 17 00:00:00 2001 From: caoqianming Date: Thu, 11 May 2023 16:20:36 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=8A=A5=E8=A1=A8?= =?UTF-8?q?=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/bi/__init__.py | 0 apps/bi/admin.py | 12 ++++++ apps/bi/apps.py | 7 +++ apps/bi/migrations/0001_initial.py | 56 ++++++++++++++++++++++++ apps/bi/migrations/__init__.py | 0 apps/bi/models.py | 15 +++++++ apps/bi/serializers.py | 38 +++++++++++++++++ apps/bi/services.py | 15 +++++++ apps/bi/tests.py | 3 ++ apps/bi/urls.py | 13 ++++++ apps/bi/views.py | 68 ++++++++++++++++++++++++++++++ apps/utils/sql.py | 54 ++++++++++++++++++++++++ server/settings.py | 5 ++- server/urls.py | 1 + 14 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 apps/bi/__init__.py create mode 100644 apps/bi/admin.py create mode 100644 apps/bi/apps.py create mode 100644 apps/bi/migrations/0001_initial.py create mode 100644 apps/bi/migrations/__init__.py create mode 100644 apps/bi/models.py create mode 100644 apps/bi/serializers.py create mode 100644 apps/bi/services.py create mode 100644 apps/bi/tests.py create mode 100644 apps/bi/urls.py create mode 100644 apps/bi/views.py create mode 100644 apps/utils/sql.py diff --git a/apps/bi/__init__.py b/apps/bi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/bi/admin.py b/apps/bi/admin.py new file mode 100644 index 00000000..f345415d --- /dev/null +++ b/apps/bi/admin.py @@ -0,0 +1,12 @@ +from django.contrib import admin +from apps.bi.models import Dataset, Report +# Register your models here. + +@admin.register(Dataset) +class DatasetAdmin(admin.ModelAdmin): + date_hierarchy = 'create_time' + + +@admin.register(Report) +class ReportAdmin(admin.ModelAdmin): + date_hierarchy = 'create_time' \ No newline at end of file diff --git a/apps/bi/apps.py b/apps/bi/apps.py new file mode 100644 index 00000000..df5c04ad --- /dev/null +++ b/apps/bi/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class BiConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.bi' + verbose_name = '报表模块' diff --git a/apps/bi/migrations/0001_initial.py b/apps/bi/migrations/0001_initial.py new file mode 100644 index 00000000..79a13b2c --- /dev/null +++ b/apps/bi/migrations/0001_initial.py @@ -0,0 +1,56 @@ +# Generated by Django 3.2.12 on 2023-05-10 05:18 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('system', '0002_myschedule'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Dataset', + fields=[ + ('id', models.CharField(editable=False, help_text='主键ID', max_length=20, primary_key=True, serialize=False, verbose_name='主键ID')), + ('create_time', models.DateTimeField(default=django.utils.timezone.now, help_text='创建时间', verbose_name='创建时间')), + ('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')), + ('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')), + ('name', models.CharField(max_length=100, verbose_name='名称')), + ('description', models.TextField(default='', verbose_name='描述说明')), + ('sql_query', models.TextField(default='', verbose_name='sql查询语句')), + ('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_belong_dept', to='system.dept', verbose_name='所属部门')), + ('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')), + ('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='dataset_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Report', + fields=[ + ('id', models.CharField(editable=False, help_text='主键ID', max_length=20, primary_key=True, serialize=False, verbose_name='主键ID')), + ('create_time', models.DateTimeField(default=django.utils.timezone.now, help_text='创建时间', verbose_name='创建时间')), + ('update_time', models.DateTimeField(auto_now=True, help_text='修改时间', verbose_name='修改时间')), + ('is_deleted', models.BooleanField(default=False, help_text='删除标记', verbose_name='删除标记')), + ('name', models.CharField(max_length=100, verbose_name='名称')), + ('code', models.CharField(default='', max_length=100, verbose_name='标识')), + ('js_function', models.TextField(default='', verbose_name='数据转化函数')), + ('belong_dept', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_belong_dept', to='system.dept', verbose_name='所属部门')), + ('create_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_create_by', to=settings.AUTH_USER_MODEL, verbose_name='创建人')), + ('datasets', models.ManyToManyField(blank=True, to='bi.Dataset', verbose_name='关联数据集')), + ('update_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='report_update_by', to=settings.AUTH_USER_MODEL, verbose_name='最后编辑人')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/apps/bi/migrations/__init__.py b/apps/bi/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/bi/models.py b/apps/bi/models.py new file mode 100644 index 00000000..bc28f5c6 --- /dev/null +++ b/apps/bi/models.py @@ -0,0 +1,15 @@ +from django.db import models +from apps.utils.models import BaseModel, CommonADModel, CommonBDModel +# Create your models here. + +class Dataset(CommonBDModel): + name = models.CharField('名称', max_length=100) + description = models.TextField('描述说明', default='', blank=True) + sql_query = models.TextField('sql查询语句', default='', blank=True) + + +class Report(CommonBDModel): + name = models.CharField('名称', max_length=100) + code = models.CharField('标识', max_length=100, default='', blank=True) + js_function = models.TextField('数据转化函数', default='', blank=True) + datasets = models.ManyToManyField(Dataset, verbose_name='关联数据集', blank=True) \ No newline at end of file diff --git a/apps/bi/serializers.py b/apps/bi/serializers.py new file mode 100644 index 00000000..c4a0c13b --- /dev/null +++ b/apps/bi/serializers.py @@ -0,0 +1,38 @@ +from apps.utils.serializers import CustomModelSerializer +from apps.bi.models import Dataset, Report +from apps.utils.constants import EXCLUDE_FIELDS +from rest_framework import serializers +from apps.bi.services import check_sql_safe + + +class DatasetCreateUpdateSerializer(CustomModelSerializer): + class Meta: + model = Dataset + exclude = EXCLUDE_FIELDS + + def validate(self, attrs): + sql_query = attrs['sql_query'] + if sql_query: + check_sql_safe(sql_query) + return attrs + +class DatasetSerializer(CustomModelSerializer): + class Meta: + model = Dataset + fields = '__all__' + + +class ReportCreateUpdateSerializer(CustomModelSerializer): + class Meta: + model = Report + exclude = EXCLUDE_FIELDS + + +class ReportSerializer(CustomModelSerializer): + class Meta: + model = Report + fields = '__all__' + + +class ReportExecSerializer(serializers.Serializer): + query = serializers.JSONField(label="查询字典参数", required=False, allow_null=True) \ No newline at end of file diff --git a/apps/bi/services.py b/apps/bi/services.py new file mode 100644 index 00000000..15cffdde --- /dev/null +++ b/apps/bi/services.py @@ -0,0 +1,15 @@ +from apps.bi.models import Dataset, Report +from apps.utils.decorators import auto_log +from rest_framework.exceptions import ParseError + +forbidden_keywords = ["UPDATE", "DELETE", "DROP", "TRUNCATE"] + + +def check_sql_safe(sql: str): + """检查sql安全性 + """ + sql_upper = sql.upper() + for kw in forbidden_keywords: + if kw in sql_upper: + raise ParseError('sql查询有风险') + return sql \ No newline at end of file diff --git a/apps/bi/tests.py b/apps/bi/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/apps/bi/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/apps/bi/urls.py b/apps/bi/urls.py new file mode 100644 index 00000000..4e2f409f --- /dev/null +++ b/apps/bi/urls.py @@ -0,0 +1,13 @@ +from django.urls import path, include +from rest_framework.routers import DefaultRouter +from apps.bi.views import ReportViewSet, DatasetViewSet + +API_BASE_URL = 'api/bi/' +HTML_BASE_URL = 'bi/' + +router = DefaultRouter() +router.register('dataset', DatasetViewSet, basename='dataset') +router.register('report', ReportViewSet, basename='report') +urlpatterns = [ + path(API_BASE_URL, include(router.urls)), +] \ No newline at end of file diff --git a/apps/bi/views.py b/apps/bi/views.py new file mode 100644 index 00000000..153761dc --- /dev/null +++ b/apps/bi/views.py @@ -0,0 +1,68 @@ +from django.shortcuts import render +from apps.utils.viewsets import CustomModelViewSet +from rest_framework.decorators import action +from rest_framework.response import Response +from apps.bi.models import Dataset, Report +from apps.bi.serializers import DatasetSerializer, DatasetCreateUpdateSerializer, ReportCreateUpdateSerializer, ReportSerializer, ReportExecSerializer +import concurrent.futures +from django.core.cache import cache +from apps.utils.sql import execute_raw_sql +from apps.bi.services import check_sql_safe +# Create your views here. + +class DatasetViewSet(CustomModelViewSet): + queryset = Dataset.objects.all() + serializer_class = DatasetSerializer + create_serializer_class = DatasetCreateUpdateSerializer + update_serializer_class = DatasetCreateUpdateSerializer + search_fields = ['name'] + + +class ReportViewSet(CustomModelViewSet): + queryset = Report.objects.all() + serializer_class = ReportSerializer + create_serializer_class = ReportCreateUpdateSerializer + update_serializer_class = ReportCreateUpdateSerializer + search_fields = ['name', 'code'] + + @action(methods=['post'], detail=True, perms_map={'post': '*'}, serializer_class=ReportExecSerializer) + def exec(self, request, pk=None): + """执行报表查询 + + 执行报表查询并用于返回前端渲染 + """ + report = self.get_object() + rdata = ReportSerializer(instance=report).data + query = request.data.get('query', {}) + query['r_user'] = request.user.id + query['r_dept'] = request.user.belong_dept.id if request.user.belong_dept else '' + datasets = report.datasets.all() + results = {} + seconds = 10 # 缓存秒数 + + with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: # 多线程运行并返回字典结果 + fun_ps = [] + for ds in datasets: + sql_query = ds.sql_query + if sql_query: + sql_f = check_sql_safe(sql_query.format(**query)) # 有风险先这样处理一下 + res = cache.get(sql_f, None) + if isinstance(res, list): + results[ds.name] = res + else: + fun_ps.append((ds.name, execute_raw_sql, sql_f, seconds)) + # 生成执行函数 + futures = {executor.submit(i[1], i[2]): i for i in fun_ps} + for future in concurrent.futures.as_completed(futures): + name, *_, sql_f, seconds = futures[future] # 获取对应的键 + try: + r = future.result() + results[name] = r + if seconds: + cache.set(sql_f, r, seconds) + except Exception as e: + results[name] = 'error: ' + str(e) + + rdata['data'] = results + return Response(rdata) + \ No newline at end of file diff --git a/apps/utils/sql.py b/apps/utils/sql.py new file mode 100644 index 00000000..3262282b --- /dev/null +++ b/apps/utils/sql.py @@ -0,0 +1,54 @@ +from django.db import connection + +def execute_raw_sql(sql: str, params=None, return_type: int =1): + """执行原始sql并返回数据 + + Args: + sql (str): 查询语句 + params (_type_, optional): 参数列表. Defaults to None. + return_type (int, optional): 返回格式. Defaults to 1. + 1 直接返回包含多个字典的列表或空列表 + 2 返回原始数据 + """ + with connection.cursor() as cursor: + if params: + cursor.execute(sql, params=params) + else: + cursor.execute(sql) + columns = [desc[0] for desc in cursor.description] + rows = cursor.fetchall() + if return_type == 1: + return [dict(zip(columns, row)) for row in rows] + else: + return rows + +def query_all_dict(sql, params=None): + ''' + 查询所有结果返回字典类型数据 + :param sql: + :param params: + :return: + ''' + with connection.cursor() as cursor: + if params: + cursor.execute(sql, params=params) + else: + cursor.execute(sql) + columns = [desc[0] for desc in cursor.description] + return [dict(zip(columns, row)) for row in cursor.fetchall()] + +def query_one_dict(sql, params=None): + """ + 查询一个结果返回字典类型数据 + :param sql: + :param params: + :return: + """ + with connection.cursor() as cursor: + if params: + cursor.execute(sql, params=params) + else: + cursor.execute(sql) + columns = [desc[0] for desc in cursor.description] + row = cursor.fetchone() + return dict(zip(columns, row)) \ No newline at end of file diff --git a/server/settings.py b/server/settings.py index 83367219..4f6396d3 100755 --- a/server/settings.py +++ b/server/settings.py @@ -29,7 +29,7 @@ DEBUG = conf.DEBUG ALLOWED_HOSTS = ['*'] SYS_NAME = 'XT_EHS' -SYS_VERSION = '2.00.13' +SYS_VERSION = '2.01.13' PROJECT_NAME = conf.PROJECT_NAME @@ -62,7 +62,8 @@ INSTALLED_APPS = [ 'apps.am', 'apps.vm', 'apps.rpm', - 'apps.opm' + 'apps.opm', + 'apps.bi' ] MIDDLEWARE = [ diff --git a/server/urls.py b/server/urls.py index 88cc5416..6c209e83 100755 --- a/server/urls.py +++ b/server/urls.py @@ -54,6 +54,7 @@ urlpatterns = [ path('', include('apps.rpm.urls')), path('', include('apps.opm.urls')), path('', include('apps.vm.urls')), + path('', include('apps.bi.urls')),