用Python寫了一個網頁抓取的工具,用來做網頁的服務器壓力測試。也可以模擬運行時的狀態。
- # coding:gbk
- import time
- import urllib2
- import threading
- from Queue import Queue
- from time import sleep
-
- # 性能測試頁面
- PERF_TEST_URL = "http://localhost:8080/perf.jsp"
-
- # 配置:壓力測試
- #THREAD_NUM = 10 # 並發線程總數
- #ONE_WORKER_NUM = 500 # 每個線程的循環次數
- #LOOP_SLEEP = 0.01 # 每次請求時間間隔(秒)
-
- # 配置:模擬運行狀態
- THREAD_NUM = 100 # 並發線程總數
- ONE_WORKER_NUM = 10000 # 每個線程的循環次數
- LOOP_SLEEP = 0.5 # 每次請求時間間隔(秒)
-
-
-
- # 出錯數
- ERROR_NUM = 0
-
-
- #具體的處理函數,負責處理單個任務
- def doWork(index):
- t = threading.currentThread()
- #print "["+t.name+" "+str(index)+"] "+PERF_TEST_URL
-
- try:
- html = urllib2.urlopen(PERF_TEST_URL).read()
- except urllib2.URLError, e:
- print "["+t.name+" "+str(index)+"] "
- print e
- global ERROR_NUM
- ERROR_NUM += 1
-
-
- #這個是工作進程,www.linuxidc.com負責不斷從隊列取數據並處理
- def working():
- t = threading.currentThread()
- print "["+t.name+"] Sub Thread Begin"
-
- i = 0
- while i < ONE_WORKER_NUM:
- i += 1
- doWork(i)
- sleep(LOOP_SLEEP)
-
- print "["+t.name+"] Sub Thread End"
-
-
- def main():
- #doWork(0)
- #return
-
- t1 = time.time()
-
- Threads = []
-
- # 創建線程
- for i in range(THREAD_NUM):
- t = threading.Thread(target=working, name="T"+str(i))
- t.setDaemon(True)
- Threads.append(t)
-
- for t in Threads:
- t.start()
-
- for t in Threads:
- t.join()
-
- print "main thread end"
-
- t2 = time.time()
- print "========================================"
- print "URL:", PERF_TEST_URL
- print "任務數量:", THREAD_NUM, "*", ONE_WORKER_NUM, "=", THREAD_NUM*ONE_WORKER_NUM
- print "總耗時(秒):", t2-t1
- print "每次請求耗時(秒):", (t2-t1) / (THREAD_NUM*ONE_WORKER_NUM)
- print "每秒承載請求數:", 1 / ((t2-t1) / (THREAD_NUM*ONE_WORKER_NUM))
- print "錯誤數量:", ERROR_NUM
-
-
- if __name__ == "__main__": main()