中国移动(中国联通)
3.整个过程，包括重建数据库表结构，解析json等如同逆向运营商的数据库一般。总体来说，三大运营商更新频率不算频繁，还算较稳定，数据结构，网页结构等都不会做很大的变动。
/*
#############################################################################################
# TongXinZhenXin Solution
# 通信征信数据抓取解决方案
# ========================================================================================= #
# File: CrawlerUNC.cs 联通抓取类
# ========================================================================================= #
# Credits: 0ng ching tong
# Thanks: ..
# Time: 2016年5月12日17:10:09
# Update: 联通最近一次抓包分析时间：2016-5-12 17:15:32
#############################################################################################
*/
using Crawler.Common;
using Crawler.Interface;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using YXRepository.Log;
using YXRepository.Model;
namespace Crawler {
public class CrawlerUNC : CrawlerBase, ICrawler {
HttpHelperNew hhn;
static IListloglist;
private string currentUVC {
get {
return HttpHelperNew.UNCuacverifykey;
}
}
////// 中国联通 初始化链接
///public CrawlerUNC(string number, string pwd) {
hhn = new HttpHelperNew;
loglist = new List;
currentPhoneNumber = number;
currentPhoneServicePwd = pwd;
loginIsNeedVerifyImgRequestUrl = "https://c.quk.cc/3/z17/5xnnuqop4i5
loginVerifyImgRequestUrl = "https://c.quk.cc/3/z17/1csfmvuy20e
loginToVerifyImgRequestUrl = "https://c.quk.cc/3/z17/4uzfv1albk1
}
public bool IsLoginNeedVerify {
string _url = loginIsNeedVerifyImgRequestUrl;
string queryData = "callback=jQuery17205245009011952871_" + TimeStamp.GetTimeStamp_13 + "&userName="+currentPhoneNumber+"&pwdType=01&_="
+ TimeStamp.GetTimeStamp_13;
string retString = hhn.HttpGet(_url, queryData, HttpForType.联通);
//添加日志记录：
CollectJsonLog(_url,queryData,0,retString);
//
return retString.Contains(@"""resultCode"":""false""") ? false : true;
}
////// 联通登录不需要图片码
/////////public bool IsLoginImgVerifyOk(string imgcode) {
currentLoginImgCode = imgcode;
string _url = loginToVerifyImgRequestUrl;
string queryData = "callback=jQuery17208163765012834383_1463034583178&verifyCode=" + currentLoginImgCode + "&verifyType=1&_=1463034805373";
string retString = hhn.HttpGet(_url, queryData);
//添加日志记录：
CollectJsonLog(_url, queryData, 0, retString);
//
return retString.Contains(@"""resultCode"":""true""") ? true : false;
}
public string GetLoginImg {
loginVerifyImgStream = string.Empty;
string queryData = "t=1463034742570";
string part1 = "data:image/png;base64,";
string part2 = hhn.HttpGetImage(loginVerifyImgRequestUrl,
queryData, HttpForType.联通);//注意：rdmdmd5这个cookie需要获取验证码后 返回。
//添加日志记录：
CollectJsonLog(loginVerifyImgRequestUrl, queryData, 0, part2);
//
if (!string.IsNullOrEmpty(part2))
loginVerifyImgStream = part1 + part2;
return loginVerifyImgStream;
}
public bool LogOut {
bool retValue = true;
return retValue;
}
public bool Login(out string loginret) {
loginret = string.Empty;
//loginRequestUrl = "https://c.quk.cc/3/z17/jxu3poaayvs" + TimeStamp.GetTimeStamp_13 +
//"&req_time=" + TimeStamp.GetTimeStamp_13 + "&redirectURL=http%3A%2F%2Fwww.10010.com&userName=" + currentPhoneNumber +
//"&password=" + currentPhoneServicePwd + "&pwdType=01&productType=01&redirectType=01&rememberMe=1&_=" + TimeStamp.GetTimeStamp_13;//服务密码登陆,不带图片验证码
loginRequestUrl = "https://c.quk.cc/3/z17/gdmt4vsjkkh" + currentPhoneNumber + "&password=" + currentPhoneServicePwd + "&pwdType=01&productType=01&verifyCode=" + currentLoginImgCode + "&uvc=" + currentUVC + "&redirectType=01&rememberMe=1&_=1463034838271";
//服务密码登陆,不带图片验证码
bool retValue = false;
string retString = string.Empty;
do {
retString = hhn.HttpGet(loginRequestUrl, "", HttpForType.联通);
Thread.Sleep(500);
}
while (retString.Contains(@"所属省份系统正在升级"));
//添加日志记录：
CollectJsonLog(loginRequestUrl,"",0,retString);
//
if (retString.Contains(@"resultCode:""7007"""))
loginret = "账户名与密码不匹配";
if (retString.Contains(@"密码出错已达上限"))
loginret = "密码出错已达上限";
if (retString.Contains(@"所属省份系统正在升级"))
loginret = "所属省份系统正在升级";
if (retString.Contains(@"resultCode:""0000"""))
retValue = true;
return retValue;
}
////// 联通独有，查账单前进行checklogin
///private bool checkLogin {
get {
string url = "http://iservice.10010.com/e3/static/check/checklogin?_="
+ TimeStamp.GetTimeStamp_13;
bool retValue = false;
string retString = hhn.HttpPost(url, "", HttpForType.联通);
//添加日志记录：
CollectJsonLog(url,"",1,retString);
//
if (retString.Contains(@"""isLogin"":true"))
retValue = true;
return retValue;
}
}
////// 联通独有
///public void UNCInitPage {
string url = "https://login.10010.com/captchazh.htm?type=05";
string retS = hhn.HttpGet(url, "", HttpForType.联通);
//添加日志记录：
CollectJsonLog(url, "", 0, retS);
//
//设置Cookie"WT_FPC"
string wt_fpc = JsHelper.GetJsMethd("GetWT_FPC", null);
CookieCollection hcc = new CookieCollection;
Cookie wtcookie = new Cookie {
Expires = DateTime.Now.AddYears(10),
Path = "/",
Domain = ".10010.com",
Name = "WT_FPC",
Value = "id=2c78d939da42319e6221460629342754:lv=1460686951978:ss=1460685811376"
//Value = wt_fpc.Substring(wt_fpc.IndexOf('=') + 1, wt_fpc.Length - 7)//此处 写死也可以，服务器不做校验。
};
hcc.Add(wtcookie);
hhn.cookie.Add(wtcookie);
}
public bool SendQuerySms {
return true;
}
////// 联通无需查询短信验证
/////////public bool IsQuerySmsVerifyOk(string smscode) {
return true;
}
public IListGetQueryData(T temp) {
return null;
}
private string getMyDetails {
if (checkLogin) {
string infoUrl = "http://iservice.10010.com/e3/static/query/searchPerInfo/?_=1464073258330&menuid=000100020001";
string retString = hhn.HttpPost(infoUrl, "", HttpForType.联通);
//添加日志记录：
CollectJsonLog(infoUrl,"",1,retString);
//
return retString;
}
return "";
}
public TXInfoModel GetInfo {
TXInfoModel tim = new TXInfoModel;
#region 第一部分
string infoUrl = "https://c.quk.cc/3/z17/0lpd2wbnmtt
string retString = hhn.HttpGet(infoUrl, "");
//添加日志记录：
CollectJsonLog(infoUrl,"",0,retString);
//
tim.CustomerName = Utilities.QuMiddle(retString, @"name"":""", @"""");
tim.CustomerSex = Utilities.QuMiddle(retString, @"sex"":""", @"""")=="1"?"男":"女";
#endregion
#region 第二部分
string retString2 = getMyDetails;
tim.Email = Utilities.QuMiddle(retString2, @"sendemail"":""", @"""");
DateTime innettime;
DateTime.TryParse(Utilities.QuMiddle(retString2, @"opendate"":""", @""""), out innettime);
tim.InNetTime = innettime;
tim.Grade = "";//星级得分
tim.IDCard = Utilities.QuMiddle(retString2, @"certnum"":""", @"""");
tim.PhoneNumber = Utilities.QuMiddle(retString2, @"usernumber"":""", @""""); ;
tim.ProviderName = "中国联通：" + Utilities.QuMiddle(retString2, @"brand"":""", @"""") + "-" + Utilities.QuMiddle(retString2, @"productName"":""", @"""");//01 ,02,03
tim.RegAddress = Utilities.QuMiddle(retString2, @"certaddr"":""", @"""");
tim.ContactNum = Utilities.QuMiddle(retString2, @"usernumber"":""", @"""");
tim.NetAge = "";
tim.PhoneStatus = Utilities.QuMiddle(retString2, @"subscrbstat"":""", @"""");
tim.RealNameInfo = Utilities.QuMiddle(retString2, @"certtype"":""", @"""");
tim.StarLevel = Utilities.QuMiddle(retString2, @"custlvl"":""", @"""");
tim.LevelInfo = "";
tim.ZipCode = "";
#endregion
#region 第三部分 话费余额/储蓄余额
string infoUrl11 = "http://iservice.10010.com/e3/static/query/accountBalance/search?_=1464858050369&menuid=000100010002";
string retString11 = hhn.HttpPost(infoUrl11, "type=onlyAccount", HttpForType.联通);
//添加日志记录：
CollectJsonLog(infoUrl11, "", 0, retString11);
//
tim.CurFee = Decimal.Parse(Utilities.QuMiddle(retString11, @"userbalance"":""", @""""));
tim.CurFeeTotal = Decimal.Parse(Utilities.QuMiddle(retString11, @"acctbalance"":""", @""""));
#endregion
#region 第三部分 积分
string infoUrl22 = "http://iservice.10010.com/e3/static/query/headerView";
string retString22 = hhn.HttpPost(infoUrl22, "", HttpForType.联通);
//添加日志记录：
CollectJsonLog(infoUrl22, "", 0, retString22);
//
int score;
int.TryParse(Utilities.QuMiddle(retString22, @"sore"":""", @""""), out score);//联通某些类型卡 返回的json中可能没有 积分这个 字段。
tim.PointValue = score;
#endregion
#region 第四部分 归属地查询
tim.PhoneAttribution = PhoneAttribution.getGuiShuDiNet(tim.PhoneNumber);
#endregion
return tim;
}
////// 获取五个月账单概括
//////public IListGetZhangDan {
IListlistZD = new List;
TXZhangDanModel temp;
string infoUrl = "http://iservice.10010.com/e3/static/query/queryHistoryBill?_="+TimeStamp.GetTimeStamp_13 +"&menuid=000100020001";
string retS = string.Empty;
string temptimes;int loopi=0;
Listtempss = GetZhangDanPostData(out temptimes);
if (checkLogin) {
foreach (var data in tempss) {
//当前月份的通话账单进行处理！
loopi++;
string PostdataS = data;
retS = hhn.HttpPost(infoUrl, PostdataS, HttpForType.联通);
//添加日志记录：
CollectJsonLog(infoUrl, PostdataS,1,retS);
//
//解析每月账单数据构建model
DateTime start1,end1;decimal d1;
string temps1= new Regex(@"""billcycle""(:)("".*?"")").Match(retS).ToString;
if (!string.IsNullOrEmpty(temps1)) {
DateTime.TryParse(Utilities.QuMiddle(temps1, @"billcycle"":""", "至"), out start1);
DateTime.TryParse(Utilities.QuMiddle(temps1, "至", @""""), out end1);
decimal.TryParse(Utilities.QuMiddle(retS, @"nowFee"":""", @""""), out d1);
temp = new TXZhangDanModel {
billStartDate = start1,
billEndDate = end1,
billFee = d1,
//（原文地址：http://www.cnblogs.com/x-poior/p/5641437.html）
};
listZD.Add(temp);
}
else {
decimal d2; decimal.TryParse(Utilities.QuMiddle(retS, @"allfee"":""", @""""), out d2);
temp = new TXZhangDanModel {
billStartDate = DateTime.Parse( temptimes[loopi-1].Split('&')[0]),
billEndDate = DateTime.Parse( temptimes[loopi-1].Split('&')[1]),
billFee = d2
};
listZD.Add(temp);
}
}
}
return listZD;
}
////// 获取详单
//////public IListGetXiangDan {
IListlistXD = new List;
TXXiangDanModel temp;
callListRequestUrl = "http://iservice.10010.com/e3/static/query/callDetail?_=" + TimeStamp.GetTimeStamp_13 + "&menuid=000100030001";
//callListRequestData = "beginDate=2016-04-01&endDate=2016-04-20&pageNo=1&pageSize=1000";
string retS = string.Empty;
Listtempss = GetXiangDanPostData;
if (checkLogin) {
foreach (var data in tempss) {
//注意，以下请求获取totalNum当月记录总数。
string PostdataS = "beginDate=" + data.Split('&')[0] + "&endDate=" + data.Split('&')[1] + "&pageNo=1&pageSize=20";
retS = hhn.HttpPost(callListRequestUrl, PostdataS, HttpForType.联通);
//添加日志记录：
CollectJsonLog(callListRequestUrl, PostdataS, 1, retS);
while (retS.Contains("暂时无法为您提供服务")) {
retS = hhn.HttpPost(callListRequestUrl, PostdataS, HttpForType.联通);
Thread.Sleep(500);
}
if (retS.Contains("系统检测您的访问过于频繁")) {
throw new Exception("访问获取详单链接过于频繁!请明天再试");
}
//注意，以下代码解析当月记录总数，获得所有确定的游标集合。
string currentMonthTotalNum = Utilities.QuMiddle(retS, @"totalRecord"":", @","""); //从上面返回串，获取当月记录总数
ListcurCurorlist = GetAllcurCuror(currentMonthTotalNum);//游标集合，翻页用.
foreach (var curcuror in curCurorlist) {
//当前月份的通话详单进行处理！
PostdataS = "beginDate=" + data.Split('&')[0] + "&endDate=" + data.Split('&')[1] + "&pageNo=" + curcuror + "&pageSize=50";
retS = hhn.HttpPost(callListRequestUrl, PostdataS, HttpForType.联通);
//添加日志记录：
CollectJsonLog(callListRequestUrl, PostdataS, 1, retS);
//
//匹配startTime date
MatchCollection stlist = new Regex(@"""calldate""(:)("".*?"")").Matches(retS);
//匹配startTime time
MatchCollection stlist2 = new Regex(@"""calltime""(:)("".*?"")").Matches(retS);
//匹配commPlac
MatchCollection cplist = new Regex(@"""homeareaName""(:)("".*?"")").Matches(retS);
//匹配commMode
MatchCollection cmlist = new Regex(@"""calltypeName""(:)("".*?"")").Matches(retS);
//匹配anotherNm
MatchCollection anlist = new Regex(@"""othernum""(:)("".*?"")").Matches(retS);
//匹配commTime
MatchCollection ctilist = new Regex(@"""calllonghour""(:)("".*?"")").Matches(retS);
//匹配commType
//MatchCollection ctylist = new Regex(@"""romatype""(:)("".*?"")").Matches(retS);
//匹配commType
MatchCollection ctylist = new Regex(@"""landtype""(:)("".*?"")").Matches(retS);
//匹配commFee
MatchCollection cflist = new Regex(@"""totalfee""(:)("".*?"")").Matches(retS);
if ((stlist.Count == cplist.Count) && (cplist.Count == cmlist.Count) && (cmlist.Count == anlist.Count)
&& (anlist.Count == ctilist.Count) && (ctilist.Count == ctylist.Count) && (ctylist.Count == cflist.Count)) {
//解析每月详单数据构建model
for (int i = 0; i < stlist.Count; i++) {
temp = new TXXiangDanModel {
anotherNm = Utilities.QuMiddle(anlist[i].Value, @"othernum"":""", @""""),
commFee = decimal.Parse(Utilities.QuMiddle(cflist[i].Value, @"totalfee"":""", @"""")),
commMode = Utilities.QuMiddle(cmlist[i].Value, @"calltypeName"":""", @""""),
commPlac = Utilities.QuMiddle(cplist[i].Value, @"homeareaName"":""", @""""),
commTime = Utilities.QuMiddle(ctilist[i].Value, @"calllonghour"":""", @""""),
commType = Utilities.QuMiddle(ctylist[i].Value, @"landtype"":""", @""""),
startTime = DateTime.Parse(Utilities.QuMiddle(stlist[i].Value, @"calldate"":""", @"""")
+ " " + Utilities.QuMiddle(stlist2[i].Value, @"calltime"":""", @""""))
};
listXD.Add(temp);
}
}
}
}
}
return listXD;
}
////// 翻页索引(1,2,3,4,5)，用于联通翻页查询账单。默认每页50条记录
//////当月份总数目///private ListGetAllcurCuror(string totalNum) {
Listretlist = new List;
int totalnum1;
int.TryParse(totalNum, out totalnum1);
if (totalnum1 == 0)
return retlist;
else {
int yushu = totalnum1 % 50;//比如totalNum 201,余数1
int curcurorCount = totalnum1 / 50 + (yushu == 0 ? 0 : 1);//5页
for (int i = 0; i < curcurorCount; i++) {
retlist.Add((i+1).ToString);//1,2,3,4,5
}
return retlist;
}
}
////// 最近5个月账单需要的Post数据
//////private ListGetZhangDanPostData(out string[] startendS) {
// string dataS = "billdate=201604&querycode=0001&querytype=0001";
Listretlist = new List; startendS = new string[5];
DateTime nowtime = DateTime.Now;
for (int i = 0; i < 5; i++) {
string mm = nowtime.AddMonths((-1)*i).ToString("yyyyMM");
string dataS = "billdate="+mm+"&querycode=0001&querytype=0001";
string seS;
if(i==0)
seS = new DateTime(nowtime.AddMonths((-1) * i).Year, nowtime.AddMonths((-1) * i).Month, 1).ToString + "&" + nowtime.AddMonths((-1) * i).ToString; //整理账单起&止月份,非本月
else
seS = new DateTime(nowtime.AddMonths((-1) * i).Year, nowtime.AddMonths((-1) * i).Month, 1).ToString + "&" + new DateTime(nowtime.AddMonths((-1) * i).Year, nowtime.AddMonths((-1) * i).Month, 1).AddMonths(1).AddDays(-1).ToString; //整理账单起&止月份，本月
startendS[i] = (seS);
retlist.Add(dataS);
}
return retlist;
}
////// 获取5个月详单需要的post数据(2016-04-01&2016-04-30格式)
//////private ListGetXiangDanPostData {
Listretlist = new List;
int year = DateTime.Now.Year;//当前年
int mouth = DateTime.Now.Month;//当前月
int beforeYear = 0;
int beforeMouth = 0;
for (int i = 0; i < 5; i++) {
if (mouth <= 1 && i!=0 )//如果当前月是一月，那么年份就要减1
{
beforeYear = year - i;
beforeMouth = 12;//上个月
} else {
beforeYear = year;
beforeMouth = mouth - i;//上个月
}
string beforeMouthOneDay = beforeYear + "-" + beforeMouth + "-" + "01";//上个月第一天
string beforeMouthLastDay;
if (i != 0)
beforeMouthLastDay = beforeYear + "-" + beforeMouth + "-" + DateTime.DaysInMonth(year, beforeMouth);//上个月最后一天
else
beforeMouthLastDay = DateTime.Now.ToString("yyyy-MM-dd");
retlist.Add(DateTime.Parse(beforeMouthOneDay).ToString("yyyy-MM-dd") + "&" + DateTime.Parse(beforeMouthLastDay).ToString("yyyy-MM-dd"));
}
return retlist;
}
public static void CollectJsonLog(string url, string data, int method1, string responseS) {
string method = method1 == 1 ? "Post" : "Get";
loglist.Add(string.Format("【请求url:{0} , 请求数据：{1} , 请求方式：{2}, 返回数据：{3} 】", url, data, method, responseS));
}
public IListGetAllJsonLog {
return loglist;
}
}
}