作业爬虫

This commit is contained in:
caoqianming 2019-10-16 15:36:07 +08:00
parent 277fc87699
commit 780daadbfa
9 changed files with 357 additions and 37 deletions

View File

@ -53,20 +53,31 @@ def makeqr_area(data):
img.save(filepath)
return filepath
def drquestions(companyid,path):
def drquestions(companyid,path,userid):
wb = load_workbook(path)
sheet = wb.worksheets[0]
i = 4
user = User.objects.get(userid=userid)
qlist = ['A','B','C','D','E','F']
leveldict = {'':1,'':2,'':3}
notinlist = []
while sheet['c'+str(i)].value!=None:
type = sheet['a'+str(i)].value.replace(' ', '')
cate = sheet['b'+str(i)].value.replace(' ', '')
title = sheet['c'+str(i)].value
answerA = sheet['d'+str(i)].value
answerB = sheet['e'+str(i)].value
answerC = sheet['f'+str(i)].value
answerD = sheet['g'+str(i)].value
answerE = sheet['h'+str(i)].value
answerF = sheet['i'+str(i)].value
answer = {}
if sheet['d'+str(i)].value:
answer['A'] = sheet['d'+str(i)].value
if sheet['e'+str(i)].value:
answer['B'] = sheet['e'+str(i)].value
if sheet['f'+str(i)].value:
answer['C'] = sheet['f'+str(i)].value
if sheet['g'+str(i)].value:
answer['D'] = sheet['g'+str(i)].value
if sheet['h'+str(i)].value:
answer['E'] = sheet['h'+str(i)].value
if sheet['i'+str(i)].value:
answer['F'] = sheet['i'+str(i)].value
right = sheet['j'+str(i)].value.replace(' ', '')
resolution = sheet['k'+str(i)].value
level = sheet['l'+str(i)].value
@ -78,19 +89,74 @@ def drquestions(companyid,path):
if objs.exists():
cateobj = objs[0]
else:
cateobj = Questioncat.objects.create(usecomp__partid=companyid,name=cate)
cateobj = Questioncat.objects.create(usecomp=Partment.objects.get(partid=companyid),name=cate)
print(cateobj)
if type == '单选':
obj = Question()
obj.type = 1
obj.cate = cateobj
obj.title = title
obj.answerA = answerA
obj.answerB = answerB
obj.answerC = answerC
obj.answerD = answerD
obj.answerE = answerE
obj.answerF = answerF
pass
if Question.objects.filter(type=1,title=title,right=right,deletemark=1).exists():
notinlist.append(i)
else:
if right in ['A','B','C','D','E','F']:
obj = Question()
obj.type = 1
obj.questioncat = cateobj
obj.title = title
obj.answer=answer
obj.right = right
obj.resolution = resolution if resolution else ''
if level in leveldict:
obj.level = leveldict[level]
else:
obj.level = 1
obj.usecoms = ','+companyid+','
obj.createuser = user
obj.save()
elif type == '多选':
right = list(right)
if Question.objects.filter(type=2,title=title,right=right,deletemark=1).exists():
notinlist.append(i)
else:
if [False for c in right if c not in qlist]:
pass
else:
obj = Question()
obj.type = 2
obj.questioncat = cateobj
obj.title = title
obj.answer=answer
obj.right = right
obj.resolution = resolution if resolution else ''
if level in leveldict:
obj.level = leveldict[level]
else:
obj.level = 1
obj.usecoms = ','+companyid+','
obj.createuser = user
obj.save()
elif type == '判断':
if right == 'A' or right == '':
right = 'A'
else:
right = 'B'
if Question.objects.filter(type=3,title=title,right=right,deletemark=1).exists():
notinlist.append(i)
else:
obj = Question()
obj.type = 3
obj.questioncat = cateobj
obj.title = title
obj.answer={'A':'','B':''}
obj.right = right
obj.resolution = resolution if resolution else ''
if level in leveldict:
obj.level = leveldict[level]
else:
obj.level = 1
obj.usecoms = ','+companyid+','
obj.createuser = user
obj.save()
i = i +1
print(notinlist)
return notinlist

View File

@ -1,21 +1,24 @@
import requests
from urllib import parse
from lxml import etree
'''headers = {
headers = {
'Host': 'cx.saws.org.cn',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0',
'Accept': 'text/plain, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate',
'Content-Type':'application/x-www-form-urlencoded',
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0, no-cache',}'''
'Referer':'http://cx.saws.org.cn/cms/html/certQuery/certQuery.do?method=getCertQueryIndex&ref=ch',
'X-Requested-With':'XMLHttpRequest'}
def getTzzs(certnum,stu_name):#特种证书(身份证号,姓名,均是字符)
certtype_code='720'
stu_name=parse.quote(parse.quote(stu_name))
url = "http://cx.saws.org.cn/cms/html/certQuery/certQuery.do?method=getCertQueryResult&ref=ch&certtype_code="+certtype_code+"&certnum="+certnum+"&stu_name="+stu_name
data = requests.get(url).text
sessionId = eval(requests.post('http://cx.saws.org.cn/cms/html/certQuery/certQuery.do?method=getServerTime',headers=headers).text)['time']
url = "http://cx.saws.org.cn/cms/html/certQuery/certQuery.do?method=getCertQueryResult&ref=ch&certtype_code="+certtype_code+"&certnum="+certnum+"&stu_name="+stu_name+'&passcode=1234'+'&sessionId='+sessionId
data = requests.get(url,headers=headers).text
tree = etree.HTML(data)
e1 = tree.xpath("//th[text()='姓名']/following-sibling::td[1]/text()")
e2 = tree.xpath("//th[text()='性别']/following-sibling::td[1]/text()")
@ -35,19 +38,25 @@ def getTzzs(certnum,stu_name):#特种证书(身份证号,姓名,均是字符)
def getAqzs(certnum,stu_name):#安全证书(身份证号,姓名,均是字符)
certtype_code='720'
stu_name=parse.quote(parse.quote(stu_name))
url = "http://cx.saws.org.cn/cms/html/certQuery/certQuery.do?method=getCertQueryResult2&ref=ch&certtype_code="+certtype_code+"&certnum="+certnum+"&stu_name="+stu_name
data = requests.get(url).text
sessionId = eval(requests.post('http://cx.saws.org.cn/cms/html/certQuery/certQuery.do?method=getServerTime',headers=headers).text)['time']
url = "http://cx.saws.org.cn/cms/html/certQuery/certQuery.do?method=getCertQueryResult2&ref=ch&certtype_code="+certtype_code+"&certnum="+certnum+"&stu_name="+stu_name+'&passcode=1234'+'&sessionId='+sessionId
data = requests.get(url,headers=headers).text
tree = etree.HTML(data)
print(data)
e1 = tree.xpath("//th[text()='姓名']/following-sibling::td[1]/text()")
e2 = tree.xpath("//th[text()='性别']/following-sibling::td[1]/text()")
e3 = tree.xpath("//th[text()='资格类型']/following-sibling::td[1]/text()")
e4 = tree.xpath("//th[text()='单位类型']/following-sibling::td[1]/text()")
e5 = tree.xpath("//th[text()='发证机关']/following-sibling::td[1]/text()")
e5 = tree.xpath("//th[text()='发证机关']/following-sibling::td[1]")
e6 = tree.xpath("//th[text()='初领日期']/following-sibling::td[1]/text()")
e7 = tree.xpath("//th[text()='应复审日期']/following-sibling::td[1]/text()")
e8 = tree.xpath("//th[text()='有效期开始时间']/following-sibling::td[1]/text()")
e9 = tree.xpath("//th[text()='有效期结束时间']/following-sibling::td[1]/text()")
cdata = []
x5=[]
for i in e5:
x5.append(i.xpath("string(.)"))
print(e6)
for i in range(len(e1)):
cdata.append({'姓名':e1[i],'性别':e2[i],'资格类型':e3[i],'单位类型':e4[i],'发证机关':e5[i],'应复审日期':e7[i].strip(),'有效期开始时间':e8[i].strip(),'有效期结束时间':e9[i].strip(),'url':url})
cdata.append({'姓名':e1[i],'性别':e2[i],'资格类型':e3[i],'单位类型':e4[i],'发证机关':x5[i],'应复审日期':e7[i].strip(),'有效期开始时间':e8[i].strip(),'有效期结束时间':e9[i].strip(),'初领日期':e6[i].strip(),'url':url})
return cdata

View File

@ -237,4 +237,37 @@
$.messager.alert('提示', '请选择一道题目!');
}
}
document.getElementById('drfile').onchange = function () {
var fileObj = this.files[0];
var url = 'drapi?a=question'
var form = new FormData();
form.append("upfile", fileObj); // 文件对象
xhr = new XMLHttpRequest(); // XMLHttpRequest 对象
xhr.open("post", url, true); //post方式url为服务器请求地址true 该参数规定请求是否异步处理。
xhr.send(form)
$('#drtmdg').dialog('close')
loading('导入中。。');
xhr.onload = function (evt) { //服务断接收完文件返回的结果
var data = JSON.parse(evt.target.responseText);
removeLoading();
if (data.code == 1) {
$.messager.alert('提示', '上传成功!', 'info', function () {
$("#main").panel({ href: 'html/question', title: '题库管理' });
});
}
else if (data.code == 2) {
msg = ''
msg += '因重复部分未成功'
$.messager.alert('提示', msg, 'info', function () {
$("#main").panel({ href: 'html/question', title: '题库管理' });
});
}
else {
$.messager.alert('提示', '失败!文件内容有误');
}
document.getElementById('drfile').value = '';
}; //请求完成
xhr.onerror = function (evt) { $.messager.alert('提示', '失败!文件内容有误'); document.getElementById('drfile').value = ''; }; //请求失败
}
</script>

View File

@ -213,12 +213,12 @@
xhr.onload = function (evt) { //服务断接收完文件返回的结果
var data = JSON.parse(evt.target.responseText);
removeLoading();
if (data.code = 1) {
if (data.code == 1) {
$.messager.alert('提示', '上传成功!', 'info', function () {
$("#main").panel({ href: 'html/risk', title: '风险辨识' });
});
}
else if (data.code = 2) {
else if (data.code == 2) {
msg = ''
msg += '因重复未成功,其余成功!'
$.messager.alert('提示', msg, 'info', function () {

View File

@ -0,0 +1,186 @@
<div style="width:100%;height:5%;">
<div style="margin-left:10px;margin-top:5px">
<form id="yhbztf">
<label>起始时间</label><input id="sqssj" name="qssj" class="easyui-datebox" style="width:110px;height:22px" />
<label>结束时间</label><input id="sjssj" name="jssj" class="easyui-datebox" style="width:110px;height:22px" />
<label>发现部门</label><input id="sfxbm" name="fxbm" style="width:200px;height:22px" />
<a onclick="yhbztsearch()" class='easyui-linkbutton' iconCls='fa-search' plain=true>查询</a></form>
</div>
</div>
<div style="width:100%;height:500px;padding: 20px">
<div style="width:33%;height:80%;float: left;margin-top:20px;" align=center>
<div id="yhlbt" style="width:80%;height:100%;">
</div>
</div>
<div style="width:33%;height:80%;float: left;margin-top:20px" align=center>
<div id="yhpgt" style="width:80%;height:100%;">
</div>
</div>
<div style="width:33%;height:80%;float: left;margin-top:20px" align=center>
<div id="yhztt" style="width:80%;height:100%;">
</div>
</div>
</div>
<div id="yhqst" style="width:100%;height:400px;padding: 20px">
</div>
<div style="width: 60%; margin: 0px auto; ">
<table name="" id="fxrpmtable" class='easyui-datagrid' title="隐患发现人排名" data-options="
rownumbers:true,
singleSelect:true,
striped: true,
method:'get',
url: 'fxhandle?a=yhfxpm',
toolbar:'#fxrpmbar'" style="height:400px">
<thead>
<tr>
<th data-options="field:'fxr__userid',hidden:true">用户ID</th>
<th data-options="field:'fxr__name',align:'right'" width="30%">姓名</th>
<th data-options="field:'fxbm__partname',align:'right'" width="40%">所属部门</th>
<th data-options="field:'number',align:'right'" width="20%">填报隐患数</th>
</tr>
</thead>
</table>
<div id='fxrpmbar'>
<form id="fxrpms">
<a onclick="fxrpmsearch()" class='easyui-linkbutton' iconCls='fa-search' plain=true>查询</a>
<a onclick="fxrpmexport()" class='easyui-linkbutton' iconCls='fa-upload' plain=true>导出</a>
<div>
<label>发现部门</label><input id="sfxbm2" name="fxbm" style="width:180px;height:22px" />
<label>起始时间</label><input id="sqssj" name="qssj" class="easyui-datebox" style="width:80px;height:22px" />
<label>结束时间</label><input id="sjssj" name="jssj" class="easyui-datebox" style="width:80px;height:22px" />
</div>
</form>
</div>
<div style="height:20px"></div>
<table name="" id="zgrpmtable" class='easyui-datagrid' title="隐患整改人排名" data-options="
rownumbers:true,
singleSelect:true,
striped: true,
method:'get',
url: 'fxhandle?a=yhzgpm',
toolbar:'#zgrpmbar'" style="height:400px">
<thead>
<tr>
<th data-options="field:'zgr__userid',hidden:true">用户ID</th>
<th data-options="field:'zgr__name',align:'right'" width="30%">姓名</th>
<th data-options="field:'zgbm__partname',align:'right'" width="40%">所属部门</th>
<th data-options="field:'number',align:'right'" width="20%">整改隐患数</th>
</tr>
</thead>
</table>
<div id='zgrpmbar'>
<form id="zgrpms">
<a onclick="zgrpmsearch()" class='easyui-linkbutton' iconCls='fa-search' plain=true>查询</a>
<a onclick="zgrpmexport()" class='easyui-linkbutton' iconCls='fa-upload' plain=true>导出</a>
<div>
<label>整改人</label><input class="easyui-textbox" name="zgr" style="width:80px;height:22px" />
<label>起始时间</label><input id="sqssj" name="qssj" class="easyui-datebox" style="width:80px;height:22px" />
<label>结束时间</label><input id="sjssj" name="jssj" class="easyui-datebox" style="width:80px;height:22px" />
</div>
</form>
</div>
</div>
<script>
$("#sfxbm,#sfxbm2").combotree({
url: 'parthandle?a=tree',
editable: false,
loadFilter: function (rows) {
return convert(rows);
},
formatter: function (node) {
var s = node.text;
if (node.children) {
s += '&nbsp;<span style=\'color:blue\'>(' + node.children.length + ')</span>';
}
return s;
},
onSelect: function (node) {
$('#sfxr').combobox({
url: 'getuser?partid=' + node.id + '&a=combobox',
editable: false,
});
}
});
function fxrpmsearch() {
var querydata = $('#fxrpms').serializeJSON();
$('#fxrpmtable').datagrid('load', querydata);
}
function zgrpmsearch() {
var querydata = $('#zgrpms').serializeJSON();
$('#zgrpmtable').datagrid('load', querydata);
}
function yhbztsearch() {
var querydata = $('#yhbztf').serializeJSON();
bindmap3_o('yhlbt', querydata)
bindmap4_o('yhpgt', querydata)
bindmap8('yhztt', querydata)
bindmap7('yhqst',querydata);
}
var now = new Date();
var year = now.getFullYear();
var month = now.getMonth() + 1;
bindmap3_o('yhlbt', { 'year': year, 'month': month });
bindmap4_o('yhpgt', { 'year': year, 'month': month });
bindmap7('yhqst');
bindmap8('yhztt');
function fxrpmexport() {
var data = $('#fxrpmtable').datagrid('getData');
var datalist = data.rows;
//datalist.unshift({'fxr__userid':'用户ID','fxr__name':'姓名','fxbm__partname':'部门','number':'发现隐患数'})
for (var j = 0, len = datalist.length; j < len; j++) {
var dic = datalist[j]
for (var key in dic) {
dic["姓名"] = dic["fxr__name"]
dic["部门"] = dic["fxbm__partname"]
dic["发现隐患数"] = dic["number"]
delete dic["fxr__userid"];
delete dic["fxr__name"]
delete dic["fxbm__partname"]
delete dic["number"]
}
}
JSONToCSVConvertor(JSON.stringify(datalist), "发现排名", true);
// var postdata = {
// data:JSON.stringify(datalist)
// }
// $.ajax({
// type: "POST",
// url: "fxhandle?a=exportexcel",
// data: postdata,
// success: function (data) {
// if (data.code==1) {
// downloadurl = 'http://' + window.location.host + '/' + data.downloadurl
// window.open(downloadurl);
// } else{
// $.messager.alert('提示', '导出失败!');
// }
// }
// });
}
function zgrpmexport() {
var data = $('#zgrpmtable').datagrid('getData');
var datalist = data.rows;
for (var j = 0, len = datalist.length; j < len; j++) {
var dic = datalist[j]
for (var key in dic) {
dic["姓名"] = dic["zgr__name"]
dic["部门"] = dic["zgbm__partname"]
dic["整改隐患数"] = dic["number"]
delete dic["zgr__userid"];
delete dic["zgr__name"]
delete dic["zgbm__partname"]
delete dic["number"]
}
}
JSONToCSVConvertor(JSON.stringify(datalist), "整改排名", true);
}
</script>

View File

@ -55,9 +55,7 @@
}},
{ field: 'yxqkssj', title: '有效期开始时间', width: 80 },
{ field: 'yxqjssj', title: '有效期结束时间', width: 80 },
{ field: 'url', title: '官网地址', width: 80 ,formatter: function(value,row,index){
return '<a style="color:blue" target="_blank" href="'+row.url+'">官网地址</a>';
} },
]]
});
function addaqzs(){

View File

@ -50,6 +50,12 @@
<label>发现部门</label><input id="sfxbm2" name="fxbm" style="width:180px;height:22px" />
<label>起始时间</label><input id="sqssj" name="qssj" class="easyui-datebox" style="width:80px;height:22px" />
<label>结束时间</label><input id="sjssj" name="jssj" class="easyui-datebox" style="width:80px;height:22px" />
<label>检查类型</label><input id="sjclx" name="jclx" class="easyui-combobox" style="width:100px;height:22px" data-options="
method: 'get',
valueField:'value',
textField:'text',
url:'getdickey?dicclass=14&a=combobox'
"/>
</div>
</form>

View File

@ -85,6 +85,7 @@ urlpatterns = [
path('html/riskcheck',views.riskcheck),#风险点排查记录
path('html/risk/edit/<int:id>/',views.riskedit),
path('html/report/edit',views.reportedit),
path('riskas',views.riskas),
#html页面

View File

@ -14,7 +14,7 @@ from django.db.models import F,Count,Sum
import requests
from .tasks import send_wechatmsgs,send_wechatmsg,yjjs,gettime,yjjs_px,yjjs_yl,updateTzzs,updateAqzs,yjjs_gc,yjjs_ws
from .export import exportdoc,exportxlsx,exportyjdoc,exportsimplexlsx,exportdoc2
from .daoru import drusers,drequipments,drrisks,cal_riskact_level,cal_area_risk_level,makeqr,makeqr_train,makeqr_riskact,makeqr_examtest,makeqr_area
from .daoru import drusers,drequipments,drrisks,drquestions,cal_riskact_level,cal_area_risk_level,makeqr,makeqr_train,makeqr_riskact,makeqr_examtest,makeqr_area
from django.forms.models import model_to_dict
from .safespider import getTzzs,getAqzs
from duibiao import calsim
@ -80,6 +80,8 @@ def transstr(obj,str1,str2):
return keystr
#html页面
def riskas(req):
return render(req,'riskas.html')
def reportedit(req):
return render(req,'reportedit.html',{'year':req.GET.get('year'),'month':req.GET.get('month'),'part':req.GET.get('part')})
def riskedit(req,id):
@ -1972,6 +1974,8 @@ def fxhandle(req):
objs = objs.filter(fxsj__gte=req.GET.get('qssj'))
if req.GET.get('jssj'):
objs = objs.filter(fxsj__lte=req.GET.get('jssj'))
if req.GET.get('jclx'):
objs = objs.filter(jclx = Dickey.objects.filter(dickeyid=req.GET.get('jclx')))
# yhnum = yhnum.count()
# objs.append({'fxr__userid':i.userid,'fxr__name':i.name,'fxbm__partname':i.ubelongpart.partname,'number':yhnum})
# objs.sort(key=takeSecond,reverse=True)
@ -2991,6 +2995,23 @@ def drapi(req):
return JsonResponse({"code":1})
except:
return JsonResponse({"code":0})
elif a == 'question':
userid = req.session['userid']
companyid = getcompany(userid)
file_name = 'c'+str(companyid)+'_'+req.FILES['upfile'].name
upload_folder = 'media/temp'
if not os.path.exists(upload_folder):
os.mkdir(upload_folder)
filepath = os.path.join(upload_folder, file_name)
filepath = filepath.replace('\\','/')
with open( filepath, 'wb') as f:
f.write(req.FILES['upfile'].read())
msg = drquestions(companyid,filepath,userid)
if msg:
return JsonResponse({"code":2,"msg":msg})
else:
return JsonResponse({"code":1})