這篇文章主要介紹了基于python3抓取pinpoint應用信息入庫,文中通過示例代碼介紹的非常詳細，對大家的學習或者工作具有一定的參考學習價值,需要的朋友可以參考下
Pinpoint是用Java編寫的大型分布式系統的APM（應用程序性能管理）工具。 受Dapper的啟發，Pinpoint提供了一種解決方案，通過在分布式應用程序中跟蹤事務來幫助分析系統的整體結構以及它們中的組件之間的相互關系.
pinpoint api:
- /applications.pinpoint 獲取applications基本信息
- /getAgentList.pinpoint 獲取對應application agent信息
- /getServerMapData.pinpoint 獲取對應app 基本數據流信息
db.py
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
|
import mysql.connectorclass MyDB(object): """docstring for MyDB""" def __init__(self, host, user, passwd , db): self.host = host self.user = user self.passwd = passwd self.db = db self.connect = None self.cursor = None def db_connect(self): """數據庫連接 """ self.connect = mysql.connector.connect(host=self.host, user=self.user, passwd=self.passwd, database=self.db) return self def db_cursor(self): if self.connect is None: self.connect = self.db_connect() if not self.connect.is_connected(): self.connect = self.db_connect() self.cursor = self.connect.cursor() return self def get_rows(self , sql): """ 查詢數據庫結果 :param sql: SQL語句 :param cursor: 數據庫游標 """ self.cursor.execute(sql) return self.cursor.fetchall() def db_execute(self, sql): self.cursor.execute(sql) self.connect.commit() def db_close(self): """關閉數據庫連接和游標 :param connect: 數據庫連接實例 :param cursor: 數據庫游標 """ if self.connect: self.connect.close() if self.cursor: self.cursor.close() |
pinpoint.py:
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
# -*- coding: utf-8 -*-'''Copyright (c) 2018, mersapAll rights reserved.摘 要: pinpoint.py創 建 者: mersap創建日期: 2019-01-17'''import sysimport requestsimport timeimport datetimeimport jsonsys.path.append('../Golf')import db #db.pyFrom_Time = datetime.datetime.now() + datetime.timedelta(seconds=-60)To_Time = datetime.datetime.now()From_TimeStamp = int(time.mktime(From_Time.timetuple()))*1000To_TimeStamp = int(time.mktime(datetime.datetime.now().timetuple()))*1000class PinPoint(object): """docstring for PinPoint""" def __init__(self, db): self.db = db super(PinPoint, self).__init__() """獲取pinpoint中應用""" def get_applications(self): '''return application dict ''' applicationListUrl = PPURL + "/applications.pinpoint" res = requests.get(applicationListUrl) if res.status_code != 200: print("請求異常,請檢查") return applicationLists = [] for app in res.json(): applicationLists.append(app) applicationListDict={} applicationListDict["applicationList"] = applicationLists return applicationListDict def getAgentList(self, appname): AgentListUrl = PPURL + "/getAgentList.pinpoint" param = { 'application':appname } res = requests.get(AgentListUrl, params=param) if res.status_code != 200: print("請求異常,請檢查") return return len(res.json().keys()),json.dumps(list(res.json().keys())) def update_servermap(self, appname , from_time=From_TimeStamp, to_time=To_TimeStamp, serviceType='SPRING_BOOT'): '''更新app上下游關系 :param appname: 應用名稱 :param serviceType: 應用類型 :param from_time: 起始時間 :param to_time: 終止時間 : ''' #https://pinpoint.*****.com/getServerMapData.pinpoint?applicationName=test-app&from=1547721493000&to=1547721553000&callerRange=1&calleeRange=1&serviceTypeName=TOMCAT&_=1547720614229 param = { 'applicationName':appname, 'from':from_time, 'to':to_time, 'callerRange':1, 'calleeRange':1, 'serviceTypeName':serviceType } # serverMapUrl = PPURL + "/getServerMapData.pinpoint" serverMapUrl = "{}{}".format(PPURL, "/getServerMapData.pinpoint") res = requests.get(serverMapUrl, params=param) if res.status_code != 200: print("請求異常,請檢查") return update_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) links = res.json()["applicationMapData"]["linkDataArray"] for link in links : ###排除test的應用 if link['sourceInfo']['applicationName'].startswith('test'): continue #應用名稱、應用類型、下游應用名稱、下游應用類型、應用節點數、下游應用節點數、總請求數、 錯誤請求數、慢請求數(本應用到下一個應用的數量) application = link['sourceInfo']['applicationName'] serviceType = link['sourceInfo']['serviceType'] to_application = link['targetInfo']['applicationName'] to_serviceType = link['targetInfo']['serviceType'] agents = len(link.get('fromAgent',' ')) to_agents = len(link.get('toAgent',' ')) totalCount = link['totalCount'] errorCount = link['errorCount'] slowCount = link['slowCount'] sql = """ REPLACE into application_server_map (application, serviceType, agents, to_application, to_serviceType, to_agents, totalCount, errorCount,slowCount, update_time, from_time, to_time) VALUES ("{}", "{}", {}, "{}", "{}", {}, {}, {}, {},"{}","{}", "{}")""".format( application, serviceType, agents, to_application, to_serviceType, to_agents, totalCount, errorCount, slowCount, update_time, From_Time, To_Time) self.db.db_execute(sql) def update_app(self): """更新application """ appdict = self.get_applications() apps = appdict.get("applicationList") update_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) for app in apps: if app['applicationName'].startswith('test'): continue agents, agentlists = self.getAgentList(app['applicationName']) sql = """ REPLACE into application_list( application_name, service_type, code, agents, agentlists, update_time) VALUES ("{}", "{}", {}, {}, '{}', "{}");""".format( app['applicationName'], app['serviceType'], app['code'], agents, agentlists, update_time) self.db.db_execute(sql) return True def update_all_servermaps(self): """更新所有應用數 """ appdict = self.get_applications() apps = appdict.get("applicationList") for app in apps: self.update_servermap(app['applicationName'], serviceType=app['serviceType']) ###刪除7天前數據 Del_Time = datetime.datetime.now() + datetime.timedelta(days=-7) sql = """delete from application_server_map where update_time <= "{}" """.format(Del_Time) self.db.db_execute(sql) return Truedef connect_db(): """ 建立SQL連接 """ mydb = db.MyDB( host="rm-*****.mysql.rds.aliyuncs.com", user="user", passwd="passwd", db="pinpoint" ) mydb.db_connect() mydb.db_cursor() return mydbdef main(): db = connect_db() pp = PinPoint(db) pp.update_app() pp.update_all_servermaps() db.db_close()if __name__ == '__main__': main() |
附sql語句
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
|
CREATE TABLE `application_list` ( `application_name` varchar(32) NOT NULL, `service_type` varchar(32) DEFAULT NULL COMMENT '服務類型', `code` int(11) DEFAULT NULL COMMENT '服務類型代碼', `agents` int(11) DEFAULT NULL COMMENT 'agent個數', `agentlists` varchar(256) DEFAULT NULL COMMENT 'agent list', `update_time` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '更新時間', PRIMARY KEY (`application_name`), UNIQUE KEY `Unique_App` (`application_name`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='pinpoint app list'CREATE TABLE `application_server_map` ( `application` varchar(32) NOT NULL COMMENT '應用名稱', `serviceType` varchar(8) NOT NULL, `agents` int(2) NOT NULL COMMENT 'agent個數', `to_application` varchar(32) NOT NULL COMMENT '下游服務名稱', `to_serviceType` varchar(32) DEFAULT NULL COMMENT '下游服務類型', `to_agents` int(2) DEFAULT NULL COMMENT '下游服務agent數量', `totalCount` int(8) DEFAULT NULL COMMENT '總請求數', `errorCount` int(8) DEFAULT NULL, `slowCount` int(8) DEFAULT NULL, `update_time` datetime NOT NULL ON UPDATE CURRENT_TIMESTAMP, `from_time` datetime DEFAULT NULL, `to_time` datetime DEFAULT NULL, PRIMARY KEY (`application`,`to_application`), UNIQUE KEY `Unique_AppMap` (`application`,`to_application`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='應用鏈路數據' |
以上就是本文的全部內容，希望對大家的學習有所幫助，也希望大家多多支持服務器之家。
原文鏈接：https://www.cnblogs.com/yum777/p/12155632.html
