Skip to content

Commit 569e2b5

Browse files
ilyam8ilyam8
committed
switch to urllib3 UrlService
1 parent b18b658 commit 569e2b5

1 file changed

Lines changed: 66 additions & 81 deletions

File tree

‎python.d/python_modules/base.py‎

Lines changed: 66 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -17,38 +17,33 @@
1717
#
1818
# using ".encode()" in one thread can block other threads as well (only in python2)
1919

20-
import time
2120
import os
21+
import re
2222
import socket
23+
import time
2324
import threading
24-
import ssl
25+
26+
import urllib3
27+
28+
from glob import glob
2529
from subprocess import Popen, PIPE
2630
from sys import exc_info
27-
from glob import glob
28-
import re
29-
try:
30-
from urlparse import urlparse
31-
except ImportError:
32-
from urllib.parse import urlparse
33-
try:
34-
import urllib.request as urllib2
35-
except ImportError:
36-
import urllib2
31+
3732
try:
3833
import MySQLdb
39-
PYMYSQL = True
34+
PY_MYSQL = True
4035
except ImportError:
4136
try:
4237
import pymysql as MySQLdb
43-
PYMYSQL = True
38+
PY_MYSQL = True
4439
except ImportError:
45-
PYMYSQL = False
40+
PY_MYSQL = False
41+
4642
import msg
4743

48-
try:
49-
PATH = os.getenv('PATH').split(':')
50-
except AttributeError:
51-
PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'.split(':')
44+
45+
PATH = os.getenv('PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin').split(':')
46+
urllib3.disable_warnings()
5247

5348

5449
# class BaseService(threading.Thread):
@@ -504,75 +499,64 @@ def __init__(self, configuration=None, name=None):
504499
self.url = self.configuration.get('url')
505500
self.user = self.configuration.get('user')
506501
self.password = self.configuration.get('pass')
507-
self.ss_cert = self.configuration.get('ss_cert')
508-
self.proxy = self.configuration.get('proxy')
502+
self.proxy_url = self.configuration.get('proxy')
503+
self._manager = None
509504

510-
def __add_openers(self, user=None, password=None, ss_cert=None, proxy=None, url=None):
505+
def __make_headers(self, user=None, password=None, proxy_url=None):
511506
user = user or self.user
512507
password = password or self.password
513-
ss_cert = ss_cert or self.ss_cert
514-
proxy = proxy or self.proxy
515-
516-
handlers = list()
517-
518-
# HTTP Basic Auth handler
519-
if all([user, password, isinstance(user, str), isinstance(password, str)]):
520-
url = url or self.url
521-
url_parse = urlparse(url)
522-
top_level_url = '://'.join([url_parse.scheme, url_parse.netloc])
523-
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
524-
passman.add_password(None, top_level_url, user, password)
525-
handlers.append(urllib2.HTTPBasicAuthHandler(passman))
526-
self.debug("Enabling HTTP basic auth")
527-
528-
# HTTPS handler
529-
# Self-signed certificate ignore
530-
if ss_cert:
531-
try:
532-
ctx = ssl.create_default_context()
533-
ctx.check_hostname = False
534-
ctx.verify_mode = ssl.CERT_NONE
535-
except AttributeError:
536-
self.error('HTTPS self-signed certificate ignore not enabled')
537-
else:
538-
handlers.append(urllib2.HTTPSHandler(context=ctx))
539-
self.debug("Enabling HTTP self-signed certificate ignore")
540-
541-
# PROXY handler
542-
if proxy and isinstance(proxy, str) and not ss_cert:
543-
handlers.append(urllib2.ProxyHandler(dict(http=proxy)))
544-
self.debug("Enabling HTTP proxy handler (%s)" % proxy)
545-
546-
opener = urllib2.build_opener(*handlers)
547-
return opener
548-
549-
def _build_opener(self, **kwargs):
508+
proxy_url = proxy_url or self.proxy_url
509+
params = dict(keep_alive=True)
510+
if user and password:
511+
key = 'basic_auth' if not proxy_url else 'proxy_basic_auth'
512+
params[key] = '{user}:{password}'.format(user=user, password=password)
513+
return urllib3.make_headers(**params)
514+
515+
def _build_header(self, **header_kw):
550516
try:
551-
return self.__add_openers(**kwargs)
517+
return self.__make_headers(**header_kw)
552518
except TypeError as error:
553-
self.error('build_opener() error:', str(error))
519+
self.error('build_header() error: {error}'.format(error=error))
520+
return None
521+
522+
def _build_manager(self, proxy_url=None, header_params=None):
523+
header_params = header_params or dict()
524+
header_params['proxy_url'] = proxy_url or self.proxy_url
525+
header = self._build_header(**header_params)
526+
if not header:
527+
return None
528+
if header_params['proxy_url']:
529+
manager = urllib3.ProxyManager
530+
params = dict(proxy_url=header_params['proxy_url'], proxy_headers=header)
531+
else:
532+
manager = urllib3.PoolManager
533+
params = dict(headers=header)
534+
try:
535+
return manager(**params)
536+
except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error:
537+
self.error('build_manager() error:', str(error))
554538
return None
555539

556-
def _get_raw_data(self, url=None, opener=None):
540+
def _get_raw_data(self, url=None, manager=None):
557541
"""
558542
Get raw data from http request
559543
:return: str
560544
"""
561-
data = None
562545
try:
563-
opener = opener or self.opener
564-
data = opener.open(url or self.url, timeout=self.update_every * 2)
565-
raw_data = data.read().decode('utf-8', 'ignore')
566-
except urllib2.URLError as error:
567-
self.error('Url: %s. Error: %s' % (url or self.url, str(error)))
568-
return None
569-
except Exception as error:
570-
self.error(str(error))
546+
url = url or self.url
547+
manager = manager or self._manager
548+
response = manager.request(method='GET',
549+
url=url,
550+
timeout=1,
551+
retries=1,
552+
headers=manager.headers)
553+
except (urllib3.exceptions.HTTPError, TypeError, AttributeError) as error:
554+
self.error('Url: {url}. Error: {error}'.format(url=url, error=error))
571555
return None
572-
finally:
573-
if data is not None:
574-
data.close()
575-
return raw_data or None
556+
if response.status == 200:
557+
return response.data.decode() or None
558+
self.error('Url: {url}. Http response status code: {code}'.format(url=url, code=response.status))
559+
return None
576560

577561
def check(self):
578562
"""
@@ -583,20 +567,21 @@ def check(self):
583567
self.error('URL is not defined or type is not <str>')
584568
return False
585569

586-
self.opener = self.__add_openers()
570+
self._manager = self._build_manager()
571+
if not self._manager:
572+
return False
587573

588574
try:
589575
data = self._get_data()
590576
except Exception as error:
591-
self.error('_get_data() failed. Url: %s. Error: %s' % (self.url, error))
577+
self.error('_get_data() failed. Url: {url}. Error: {error}'.format(url=self.url, error=error))
592578
return False
593579

594580
if isinstance(data, dict) and data:
595581
self._data_from_check = data
596582
return True
597-
else:
598-
self.error("_get_data() returned no data or type is not <dict>")
599-
return False
583+
self.error('_get_data() returned no data or type is not <dict>')
584+
return False
600585

601586

602587
class SocketService(SimpleService):
@@ -1046,7 +1031,7 @@ def is_valid_query(query):
10461031
log_error('Unsupported "queries" format. Must be not empty <dict>')
10471032
return None
10481033

1049-
if not PYMYSQL:
1034+
if not PY_MYSQL:
10501035
self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin')
10511036
return False
10521037

0 commit comments

Comments
 (0)