From 78de6199cef83073421f4d3d59daf39b698c7dd2 Mon Sep 17 00:00:00 2001 From: nico Date: Fri, 26 Jun 2020 19:08:22 +0200 Subject: initial ci runner config * have mercy on my soul --- .gitlab-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..3c9fb21 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,16 @@ +image: "python:3.7" + +# setup environemnt +before_script: + - python --version + - pip install -r requirements.txt + - pip install flake8 + +stages: + - Static Analysis + +flake8: + stage: Static Analysis + script: + - flake8 --count --select=E9,F63,F7,F82 --show-source --statistics + - flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics -- cgit v1.2.3-18-g5258 From 2f730bf186da8a52b0b13040bd83949ee1366cfe Mon Sep 17 00:00:00 2001 From: nico Date: Fri, 26 Jun 2020 23:05:21 +0200 Subject: flake8 gitlab ajustments * ajust ci config for gitlabs ci * reduced max line length to 120 It seems gitlab prefers it that way + add some comments --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c9fb21..63ed231 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,10 +7,12 @@ before_script: - pip install flake8 stages: - - Static Analysis + - check flake8: - stage: Static Analysis + stage: check script: - - flake8 --count --select=E9,F63,F7,F82 --show-source --statistics - - flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # breaking errors and syntax errors + - flake8 --select=E9,F63,F7,F82 --show-source + # pep8 warnings + - flake8 --max-complexity=10 --max-line-length=120 --show-source -- cgit v1.2.3-18-g5258 From e503508b5b1b14dd61a1426b798837f873ba772e Mon Sep 17 00:00:00 2001 From: nico Date: Fri, 26 Jun 2020 23:31:15 +0200 Subject: split up flake jobs * split flake8 jobs into two seperate jobs to better control the result + add allow_failure: true to the pep8 job to not discourage anybody from commiting code --- .gitlab-ci.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 63ed231..5189dd4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,12 +7,18 @@ before_script: - pip install flake8 stages: - - check + - syntax + - pep8 -flake8: - stage: check +syntax: + stage: syntax script: - # breaking errors and syntax errors + # breaking errors ie syntax errors - flake8 --select=E9,F63,F7,F82 --show-source - # pep8 warnings + +pep8: + stage: pep8 + script: + # pep8 warnings - flake8 --max-complexity=10 --max-line-length=120 --show-source + allow_failure: true -- cgit v1.2.3-18-g5258 From 453d054ebbab62080c1483d6d160e251f4841155 Mon Sep 17 00:00:00 2001 From: nico Date: Sat, 27 Jun 2020 00:33:10 +0200 Subject: gitlab ci caching + add pip caching + add venv caching --- .gitlab-ci.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5189dd4..f99e762 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,23 @@ image: "python:3.7" +variables: + # force pip cache dir + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + +cache: + paths: + # utilize pip caching + - .cache/pip + # cache the virtualenv to reduce load + - venv/ + # setup environemnt before_script: - python --version + - pip install virtualenv + - virtualenv venv + - source venv/bin/activate - pip install -r requirements.txt - - pip install flake8 stages: - syntax @@ -12,6 +25,8 @@ stages: syntax: stage: syntax + before_script: + - pip install flake8 script: # breaking errors ie syntax errors - flake8 --select=E9,F63,F7,F82 --show-source -- cgit v1.2.3-18-g5258 From 565702d91f67b0b94c327574aa89564a40b95a37 Mon Sep 17 00:00:00 2001 From: nico Date: Sat, 27 Jun 2020 00:40:18 +0200 Subject: gitlab ci syntax * multiple before_script segments overwrite one another --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f99e762..87a0c5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ cache: paths: # utilize pip caching - .cache/pip - # cache the virtualenv to reduce load + # cache the virtual environment - venv/ # setup environemnt @@ -18,6 +18,7 @@ before_script: - virtualenv venv - source venv/bin/activate - pip install -r requirements.txt + - pip install flake8 stages: - syntax @@ -25,8 +26,6 @@ stages: syntax: stage: syntax - before_script: - - pip install flake8 script: # breaking errors ie syntax errors - flake8 --select=E9,F63,F7,F82 --show-source -- cgit v1.2.3-18-g5258 From fa87c166ed68fac05970ae60eda532d6ba5a851a Mon Sep 17 00:00:00 2001 From: nico Date: Sat, 27 Jun 2020 00:56:38 +0200 Subject: finalizing the ci pipelines - revert 565702d9 as it did not behave like it should have + add black pipeline for code consistency --- .gitlab-ci.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87a0c5a..c39fb0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,22 +6,17 @@ variables: cache: paths: - # utilize pip caching - - .cache/pip - # cache the virtual environment - - venv/ + - .cache/pip # pip caching directory # setup environemnt before_script: - python --version - - pip install virtualenv - - virtualenv venv - - source venv/bin/activate - pip install -r requirements.txt - - pip install flake8 + - pip install flake8 black stages: - syntax + - black - pep8 syntax: @@ -30,9 +25,15 @@ syntax: # breaking errors ie syntax errors - flake8 --select=E9,F63,F7,F82 --show-source +black: + stage: black + script: + # code consistency + - black . --check --line-length 120 + pep8: stage: pep8 script: - # pep8 warnings + # pep8 warnings and other non breaking warnings - flake8 --max-complexity=10 --max-line-length=120 --show-source allow_failure: true -- cgit v1.2.3-18-g5258 From 8b26b1eb8e5cd8d69e600a4c4eec9bea28ef8059 Mon Sep 17 00:00:00 2001 From: nico Date: Sat, 27 Jun 2020 12:07:09 +0200 Subject: tool configuration * seperate tool config and tool call * rename black stage to code consistency for a greater readability - remove allow_failure from the third stage I did decide to remove the allowed failure tag to encourage a higher code quality. --- .gitlab-ci.yml | 17 ++++++++--------- pyproject.toml | 20 ++++++++++++++++++++ setup.cfg | 5 +++++ 3 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 pyproject.toml create mode 100644 setup.cfg diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c39fb0b..5f1e754 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,24 +16,23 @@ before_script: stages: - syntax - - black - - pep8 + - code consistency + - flake8 syntax: stage: syntax script: - # breaking errors ie syntax errors + # flake8 check only breaking errors ie syntax errors - flake8 --select=E9,F63,F7,F82 --show-source black: - stage: black + stage: code consistency script: # code consistency - - black . --check --line-length 120 + - black . --check pep8: - stage: pep8 + stage: flake8 script: - # pep8 warnings and other non breaking warnings - - flake8 --max-complexity=10 --max-line-length=120 --show-source - allow_failure: true + # full flake8 test + - flake8 --show-source diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e834e5c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[tool.black] +line-length = 120 +target-version = ['py37', 'py38'] +include = '\.pyi?$' +exclude = ''' +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ # the root of the project +) +''' diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6e6bed8 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[flake8] +ignore = E501 +exclude = .git,__pycache__,.gitlab +max-complexity = 15 +max-line-length = 120 -- cgit v1.2.3-18-g5258 From a8b78be75d44dedf4b7fdf2eacb4d8a32e0b3aef Mon Sep 17 00:00:00 2001 From: nico Date: Mon, 29 Jun 2020 01:30:33 +0200 Subject: black flake8 compatability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit + add E203 whitespace before ‘:’ to flake8 ignore Due to black's formatting we need to ignore flake8 warning. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6e6bed8..76eda04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -ignore = E501 +ignore = E501,E203 exclude = .git,__pycache__,.gitlab max-complexity = 15 max-line-length = 120 -- cgit v1.2.3-18-g5258 From 5f0cb289c1d91513427ffdc8f646bc86516c2759 Mon Sep 17 00:00:00 2001 From: nico Date: Mon, 29 Jun 2020 11:03:49 +0200 Subject: black reformat * black reformatted the code --- api.py | 5 ++-- calls.py | 10 +++---- cleanup.py | 36 ++++++++++++------------ config.py | 10 +++---- influx.py | 57 +++++++++++++++++++++----------------- metrics.py | 29 +++++++++---------- prometheus.py | 89 ++++++++++++++++++++++++++++++++++++++--------------------- 7 files changed, 136 insertions(+), 100 deletions(-) diff --git a/api.py b/api.py index b14fa52..3372976 100644 --- a/api.py +++ b/api.py @@ -9,6 +9,7 @@ class EjabberdApi: """ class to interact with the ejabberd rest/ xmlrpc api """ + def __init__(self, url, login=None, api: str = "rpc"): # api variables self._login = login @@ -25,7 +26,7 @@ class EjabberdApi: @property def _auth(self) -> (str, None): if self._login is not None: - return f"{self._login['user']}@{self._login['server']}", self._login['password'] + return f"{self._login['user']}@{self._login['server']}", self._login["password"] return None def _rest(self, command: str, data) -> dict: @@ -34,7 +35,7 @@ class EjabberdApi: self.session.auth = self._auth # post - r = self.session.post('/'.join([self._url, command]), json=data) + r = self.session.post("/".join([self._url, command]), json=data) # proceed if response is ok if r.ok: diff --git a/calls.py b/calls.py index 8af5e89..8c615b8 100644 --- a/calls.py +++ b/calls.py @@ -14,8 +14,8 @@ class EjabberdApiCalls(EjabberdApi): @property def nodename(self): if self._login is not None: - node_str = re.compile('The node \'(.*)\'') - status = self.cmd('status', {}) + node_str = re.compile("The node '(.*)'") + status = self.cmd("status", {}) # matches try: @@ -32,8 +32,8 @@ class EjabberdApiCalls(EjabberdApi): @property def verstring(self): if self._login is not None: - ver_str = re.compile('([1-9][0-9.]+(?![.a-z]))\\b') - status = self.cmd('status', {}) + ver_str = re.compile("([1-9][0-9.]+(?![.a-z]))\\b") + status = self.cmd("status", {}) # matches try: @@ -117,7 +117,7 @@ class EjabberdApiCalls(EjabberdApi): host = "global" if vhost is not None: if self.verstring.major >= 19: - host = '.'.join([muc_host, vhost]) + host = ".".join([muc_host, vhost]) else: host = vhost result = self.cmd("muc_online_rooms", {"host": host}) diff --git a/cleanup.py b/cleanup.py index f66bea9..ee9def1 100755 --- a/cleanup.py +++ b/cleanup.py @@ -20,16 +20,16 @@ class EjabberdCleanup(EjabberdApiCalls): if self.dry: logging.warning(f"{user}@{host}: dry delete : {reason}") else: - self.cmd("unregister", {"host": host, "user": user}) + self.cmd("unregister", {"host": host, "user": user}) logging.warning(f"{user}@{host}: deleted - {reason}") def run_user(self, host, user): if self.skip_by_roster: - roster = self.cmd("get_roster",{"host": host, "user": user}) + roster = self.cmd("get_roster", {"host": host, "user": user}) if len(roster) > 0: logging.debug(f"{user}@{host}: skipped it has a roster") return - last = self.cmd("get_last",{"host": host, "user": user}) + last = self.cmd("get_last", {"host": host, "user": user}) if self.delete_not_login and last["status"] == "Registered but didn't login": self.delete_user(host, user, "not login") return @@ -37,14 +37,16 @@ class EjabberdCleanup(EjabberdApiCalls): last_stamp = last["timestamp"] lastdate = None try: - lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%SZ") + lastdate = datetime.datetime.strptime(last_stamp, "%Y-%m-%dT%H:%M:%SZ") except: try: - lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%S.%fZ") + lastdate = datetime.datetime.strptime(last_stamp, "%Y-%m-%dT%H:%M:%S.%fZ") except: logging.error(f"{user}@{host}: not able to parse '{last_stamp}'") return - if lastdate != None and lastdate-datetime.datetime.now() > datetime.timedelta(days=self.offline_since_days): + if lastdate != None and lastdate - datetime.datetime.now() > datetime.timedelta( + days=self.offline_since_days + ): self.delete_user(host, user, f"last seen @ {lastdate}") return @@ -53,7 +55,7 @@ class EjabberdCleanup(EjabberdApiCalls): if host in self.ignore_hosts: continue logging.info(f"run cleanup for host: {host}") - for user in self.cmd("registered_users",{"host": host}): + for user in self.cmd("registered_users", {"host": host}): if user in self.ignore_usernames: continue self.run_user(host, user) @@ -64,20 +66,20 @@ if __name__ == "__main__": # load config config = Config() - if config.get('debug', default=False): + if config.get("debug", default=False): logging.getLogger().setLevel(logging.DEBUG) # credentials and parameters - url = config.get('url', default='http://localhost:5280/api') - login = config.get('login', default=None) - api = config.get('api', default='rest') + url = config.get("url", default="http://localhost:5280/api") + login = config.get("login", default=None) + api = config.get("api", default="rest") # init handler cleaner = EjabberdCleanup(url, login, api) - cleaner.dry = config.get('cleaner_dry',default=False) - cleaner.ignore_hosts = config.get('cleaner_ignore_hosts',default=[]) - cleaner.ignore_usernames = config.get('cleaner_ignore_usernames',default=[]) - cleaner.skip_by_roster = config.get('cleaner_skip_by_roster',default=True) - cleaner.delete_not_login = config.get('cleaner_delete_not_login',default=True) - cleaner.offline_since_days = config.get('cleaner_offline_since_days', default=None) + cleaner.dry = config.get("cleaner_dry", default=False) + cleaner.ignore_hosts = config.get("cleaner_ignore_hosts", default=[]) + cleaner.ignore_usernames = config.get("cleaner_ignore_usernames", default=[]) + cleaner.skip_by_roster = config.get("cleaner_skip_by_roster", default=True) + cleaner.delete_not_login = config.get("cleaner_delete_not_login", default=True) + cleaner.offline_since_days = config.get("cleaner_offline_since_days", default=None) cleaner.run() diff --git a/config.py b/config.py index 20793e8..d4a7097 100644 --- a/config.py +++ b/config.py @@ -13,11 +13,11 @@ class Config: def __init__(self): # class variables self.content = None - self.conf_file = Path('/etc/ejabberd-metrics.yml') + self.conf_file = Path("/etc/ejabberd-metrics.yml") # dev config overwrite - if environ.get('ejabberd_metrics_dev'): - self.conf_file = Path('config.yml') + if environ.get("ejabberd_metrics_dev"): + self.conf_file = Path("config.yml") # read config file self._read() @@ -27,9 +27,9 @@ class Config: self._check() # open file as an iostream - with open(self.conf_file, 'r', encoding='utf-8') as f: + with open(self.conf_file, "r", encoding="utf-8") as f: try: - self.content = YAML(typ='safe').load(f) + self.content = YAML(typ="safe").load(f) # catch json decoding errors except (ParserError, ScannerError) as err: diff --git a/influx.py b/influx.py index e5b0065..b51d8f5 100644 --- a/influx.py +++ b/influx.py @@ -21,8 +21,8 @@ class Influx: @staticmethod def _rmspace(key: str = None, value: (str, int) = None): try: - key = key.replace(' ', '\ ') - value = value.replace(' ', '\ ') + key = key.replace(" ", "\ ") + value = value.replace(" ", "\ ") except (TypeError, AttributeError): pass @@ -36,10 +36,10 @@ class Influx: # create tag_key=tag_value pairs for all elements and append them to name for k, v in tags.items(): - output += ',{}={}'.format(*self._rmspace(k, v)) + output += ",{}={}".format(*self._rmspace(k, v)) # append key=value to name - output += ' {}={}i {}'.format(*self._rmspace(key, value), ts) + output += " {}={}i {}".format(*self._rmspace(key, value), ts) return output def write_metrics(self): @@ -47,53 +47,60 @@ class Influx: # global values cur_ts = self._timestamp() - data.append(f'ejabberd s2s_in={self._metrics.get_s2s_in()}i {cur_ts}') - data.append(f'ejabberd s2s_out={self._metrics.get_s2s_out()}i {cur_ts}') - data.append(f'ejabberd uptime={self._metrics.get_uptime()}i {cur_ts}') - data.append(f'ejabberd processes={self._metrics.get_processes()}i {cur_ts}') + data.append(f"ejabberd s2s_in={self._metrics.get_s2s_in()}i {cur_ts}") + data.append(f"ejabberd s2s_out={self._metrics.get_s2s_out()}i {cur_ts}") + data.append(f"ejabberd uptime={self._metrics.get_uptime()}i {cur_ts}") + data.append(f"ejabberd processes={self._metrics.get_processes()}i {cur_ts}") # vhost values for vhost in self._metrics.get_vhosts(): cur_ts = self._timestamp() - data.append(f'ejabberd,vhost={vhost} registered={self._metrics.get_registered(vhost)}i {cur_ts}') - data.append(f'ejabberd,vhost={vhost} muc={self._metrics.get_muc(vhost)}i {cur_ts}') + data.append(f"ejabberd,vhost={vhost} registered={self._metrics.get_registered(vhost)}i {cur_ts}") + data.append(f"ejabberd,vhost={vhost} muc={self._metrics.get_muc(vhost)}i {cur_ts}") # vhost statistics on their respective node for node in self._metrics.get_nodes(): cur_ts = self._timestamp() for k, v in self._metrics.get_online_by_status(node=node, vhost=vhost).items(): - data.append(self._parse('ejabberd_online_status', k, v, cur_ts, {'node': node, 'vhost': vhost})) + data.append(self._parse("ejabberd_online_status", k, v, cur_ts, {"node": node, "vhost": vhost})) for k, v in self._metrics.get_online_by_client(node=node, vhost=vhost).items(): - data.append(self._parse('ejabberd_online_client', k, v, cur_ts, {'node': node, 'vhost': vhost})) + data.append(self._parse("ejabberd_online_client", k, v, cur_ts, {"node": node, "vhost": vhost})) for k, v in self._metrics.get_online_by_ipversion(node=node, vhost=vhost).items(): - data.append(self._parse('ejabberd_online_ipversion', k, v, cur_ts, {'node': node, 'vhost': vhost})) + data.append(self._parse("ejabberd_online_ipversion", k, v, cur_ts, {"node": node, "vhost": vhost})) for k, v in self._metrics.get_online_by_connection(node=node, vhost=vhost).items(): - data.append(self._parse('ejabberd_online_connection', k, v, cur_ts, {'node': node, 'vhost': vhost})) + data.append(self._parse("ejabberd_online_connection", k, v, cur_ts, {"node": node, "vhost": vhost})) for cl, ipv in self._metrics.get_online_client_by_ipversion(node=node, vhost=vhost).items(): for k, v in ipv.items(): - data.append(self._parse('ejabberd_online_client_ipversion', k, v, cur_ts, - {'vhost': vhost, 'node': node, 'ipversion': k, 'client': cl})) + data.append( + self._parse( + "ejabberd_online_client_ipversion", + k, + v, + cur_ts, + {"vhost": vhost, "node": node, "ipversion": k, "client": cl}, + ) + ) # write output to database - self.client.write_points(data, time_precision='ms', batch_size=10000, protocol='line') + self.client.write_points(data, time_precision="ms", batch_size=10000, protocol="line") -if __name__ == '__main__': +if __name__ == "__main__": # load config config = Config() - if config.get('debug', default=False): + if config.get("debug", default=False): logging.getLogger().setLevel(logging.DEBUG) # credentials and parameters - url = config.get('url', default='http://localhost:5280/api') - login = config.get('login', default=None) - api = config.get('api', default='rest') + url = config.get("url", default="http://localhost:5280/api") + login = config.get("login", default=None) + api = config.get("api", default="rest") # config influxdb - influx_host = config.get('influxdb_host', default='localhost') - influx_port = config.get('influxdb_port', default=8086) - influx_dbname = config.get('influxdb_db', default='ejabberd') + influx_host = config.get("influxdb_host", default="localhost") + influx_port = config.get("influxdb_port", default=8086) + influx_dbname = config.get("influxdb_db", default="ejabberd") # init handler metrics = EjabberdMetrics(url, login, api) diff --git a/metrics.py b/metrics.py index 18d2eeb..ede3b1b 100755 --- a/metrics.py +++ b/metrics.py @@ -12,7 +12,8 @@ class EjabberdMetrics(EjabberdApiCalls): """ class to fetch metrics per xmlrpc """ - def __init__(self, url, login=None, api="rpc", muc_host: str = 'conference'): + + def __init__(self, url, login=None, api="rpc", muc_host: str = "conference"): # init ejabberd api super().__init__(url, login, api) @@ -32,7 +33,7 @@ class EjabberdMetrics(EjabberdApiCalls): "profanity": [], "Xabber": ["xabber", "xabber-android"], "ChatSecure": ["chatsecure"], - "Monal": [] + "Monal": [], } for client, names in clientmap.items(): @@ -110,7 +111,7 @@ class EjabberdMetrics(EjabberdApiCalls): return self.get_online_by("node", vhost=vhost) def get_online_by_vhost(self, node=None): - return self.get_online_by("jid", parse=lambda jid: jid[jid.find("@")+1:jid.find("/")], node=node) + return self.get_online_by("jid", parse=lambda jid: jid[jid.find("@") + 1 : jid.find("/")], node=node) def get_online_by_status(self, vhost=None, node=None): return self.get_online_by("status", vhost=vhost, node=node) @@ -187,11 +188,11 @@ class EjabberdMetrics(EjabberdApiCalls): return self._uptime def get_processes(self): - if not hasattr(self, '_processes'): + if not hasattr(self, "_processes"): self._processes = self.fetch_processes() return self._processes - def get_vhost_metrics(self, vhost): + def get_vhost_metrics(self, vhost): data = { "registered": self.get_registered(vhost), "muc": self.get_muc(vhost), @@ -199,7 +200,7 @@ class EjabberdMetrics(EjabberdApiCalls): "online_by_client": self.get_online_by_client(vhost), "online_by_ipversion": self.get_online_by_ipversion(vhost), "online_by_connection": self.get_online_by_connection(vhost), - "online_by_node": self.get_online_by_node(vhost) + "online_by_node": self.get_online_by_node(vhost), } return data @@ -209,13 +210,13 @@ class EjabberdMetrics(EjabberdApiCalls): self._nodes = self.fetch_nodes() return self._nodes - def get_node_metrics(self, node): + def get_node_metrics(self, node): data = { "online_by_status": self.get_online_by_status(node=node), "online_by_client": self.get_online_by_client(node=node), "online_by_ipversion": self.get_online_by_ipversion(node=node), "online_by_connection": self.get_online_by_connection(node=node), - "online_by_vhost": self.get_online_by_vhost(node=node) + "online_by_vhost": self.get_online_by_vhost(node=node), } return data @@ -225,7 +226,7 @@ class EjabberdMetrics(EjabberdApiCalls): "registered": self.get_registered(), "muc": self.get_muc(), "s2s_in": self.get_s2s_in(), - "s2s_out": self.get_s2s_out(), + "s2s_out": self.get_s2s_out(), "uptime": self.get_uptime(), "processes": self.get_processes(), "online_by_status": self.get_online_by_status(), @@ -233,14 +234,14 @@ class EjabberdMetrics(EjabberdApiCalls): "online_by_ipversion": self.get_online_by_ipversion(), "online_by_connection": self.get_online_by_connection(), "online_by_node": self.get_online_by_node(), - "online_by_vhost": self.get_online_by_vhost() + "online_by_vhost": self.get_online_by_vhost(), } vhosts = {} for host in self.get_vhosts(): vhosts[host] = self.get_vhost_metrics(host) data["vhosts"] = vhosts - + nodes = {} for node in self.get_nodes(): nodes[node] = self.get_node_metrics(node) @@ -259,9 +260,9 @@ if __name__ == "__main__": config = Config() # credentials and parameters - url = config.get('url', default='http://localhost:5280/api') - login = config.get('login', default=None) - api = config.get('api', default='rest') + url = config.get("url", default="http://localhost:5280/api") + login = config.get("login", default=None) + api = config.get("api", default="rest") # init handler metrics = EjabberdMetrics(url, login, api) diff --git a/prometheus.py b/prometheus.py index 9af4de3..6d7bbe5 100755 --- a/prometheus.py +++ b/prometheus.py @@ -7,9 +7,7 @@ from socket import AF_INET6 from time import time from urllib.parse import parse_qs, urlparse -from prometheus_client import ( - CollectorRegistry, Gauge, generate_latest, CONTENT_TYPE_LATEST -) +from prometheus_client import CollectorRegistry, Gauge, generate_latest, CONTENT_TYPE_LATEST from config import Config from metrics import EjabberdMetrics @@ -17,26 +15,25 @@ from metrics import EjabberdMetrics class DynamicMetricsHandler(BaseHTTPRequestHandler): """HTTP handler that gives metrics from ``core.REGISTRY``.""" + def do_GET(self): params = parse_qs(urlparse(self.path).query) registry = self.generator(params) - if 'name[]' in params: - registry = registry.restricted_registry(params['name[]']) + if "name[]" in params: + registry = registry.restricted_registry(params["name[]"]) try: output = generate_latest(registry) except: - self.send_error(500, 'error generating metric output') + self.send_error(500, "error generating metric output") raise self.send_response(200) - self.send_header('Content-Type', CONTENT_TYPE_LATEST) + self.send_header("Content-Type", CONTENT_TYPE_LATEST) self.end_headers() self.wfile.write(output) @staticmethod def factory(registry_generator): - DynMetricsHandler = type('MetricsHandler', - (DynamicMetricsHandler, object), - {"generator": registry_generator}) + DynMetricsHandler = type("MetricsHandler", (DynamicMetricsHandler, object), {"generator": registry_generator}) return DynMetricsHandler @@ -45,7 +42,7 @@ class Prometheus: self.ttl = 10 self._last_update = 0 self._metrics = metrics - + def handler(self, metrics_handler): now = time() if now >= (self._last_update + self.ttl): @@ -54,25 +51,53 @@ class Prometheus: registry = CollectorRegistry(auto_describe=True) - Gauge('ejabberd_node_s2s_in', 'count of incoming server-to-server connection', registry=registry).set(self._metrics.get_s2s_in()) - Gauge('ejabberd_node_s2s_out', 'count of outgoing server-to-server connection', registry=registry).set(self._metrics.get_s2s_out()) + Gauge("ejabberd_node_s2s_in", "count of incoming server-to-server connection", registry=registry).set( + self._metrics.get_s2s_in() + ) + Gauge("ejabberd_node_s2s_out", "count of outgoing server-to-server connection", registry=registry).set( + self._metrics.get_s2s_out() + ) nodename = self._metrics.nodename - Gauge('ejabberd_node_uptime', 'uptime of ejabberd service', ["node"], registry=registry).labels(nodename).set(self._metrics.get_uptime()) - Gauge('ejabberd_node_proccess', 'count of pejabber proccess', ["node"], registry=registry).labels(nodename).set(self._metrics.get_processes()) + Gauge("ejabberd_node_uptime", "uptime of ejabberd service", ["node"], registry=registry).labels(nodename).set( + self._metrics.get_uptime() + ) + Gauge("ejabberd_node_proccess", "count of pejabber proccess", ["node"], registry=registry).labels(nodename).set( + self._metrics.get_processes() + ) labelnames_vhost = ["vhost"] - registered_vhosts = Gauge('ejabberd_registered_vhosts', 'count of user per vhost', labelnames_vhost, registry=registry) - muc = Gauge('ejabberd_muc', 'count of muc\'s per vhost', labelnames_vhost, registry=registry) - - online_vhost_node = Gauge('ejabberd_online_vhost_node', 'count of client connections', ["vhost", "node"], registry=registry) - - online_status = Gauge('ejabberd_online_status', 'count of client connections', ["vhost", "node", "status"], registry=registry) - online_connection = Gauge('ejabberd_online_connection', 'count of client connections', ["vhost", "node", "connection"], registry=registry) - online_client = Gauge('ejabberd_online_client', 'count of client software', ["vhost", "node", "client"], registry=registry) - online_ipversion = Gauge('ejabberd_online_ipversion', 'count of client software', ["vhost", "node", "ipversion"], registry=registry) - online_client_ipversion = Gauge('ejabberd_online_client_ipversion', 'count of client software', ["vhost", "node", "client", "ipversion"], registry=registry) + registered_vhosts = Gauge( + "ejabberd_registered_vhosts", "count of user per vhost", labelnames_vhost, registry=registry + ) + muc = Gauge("ejabberd_muc", "count of muc's per vhost", labelnames_vhost, registry=registry) + + online_vhost_node = Gauge( + "ejabberd_online_vhost_node", "count of client connections", ["vhost", "node"], registry=registry + ) + + online_status = Gauge( + "ejabberd_online_status", "count of client connections", ["vhost", "node", "status"], registry=registry + ) + online_connection = Gauge( + "ejabberd_online_connection", + "count of client connections", + ["vhost", "node", "connection"], + registry=registry, + ) + online_client = Gauge( + "ejabberd_online_client", "count of client software", ["vhost", "node", "client"], registry=registry + ) + online_ipversion = Gauge( + "ejabberd_online_ipversion", "count of client software", ["vhost", "node", "ipversion"], registry=registry + ) + online_client_ipversion = Gauge( + "ejabberd_online_client_ipversion", + "count of client software", + ["vhost", "node", "client", "ipversion"], + registry=registry, + ) for host in self._metrics.get_vhosts(): labels_vhost = host @@ -108,19 +133,19 @@ class Prometheus: if __name__ == "__main__": # load config config = Config() - if config.get('debug', default=False): + if config.get("debug", default=False): logging.getLogger().setLevel(logging.DEBUG) # credentials and parameters - url = config.get('url', default='http://[::1]:5280/api') - login = config.get('login', default=None) - api = config.get('api', default='rest') + url = config.get("url", default="http://[::1]:5280/api") + login = config.get("login", default=None) + api = config.get("api", default="rest") # config prometheus - prom_addr = config.get('prometheus_address', default='127.0.0.1') - prom_port = config.get('prometheus_port', default=8080) + prom_addr = config.get("prometheus_address", default="127.0.0.1") + prom_port = config.get("prometheus_port", default=8080) metrics = EjabberdMetrics(url, login, api) prom = Prometheus(metrics) - prom.ttl = config.get('prometheus_cache_ttl', default=10) + prom.ttl = config.get("prometheus_cache_ttl", default=10) prom.listen((prom_addr, prom_port)) -- cgit v1.2.3-18-g5258 From 044e0334d2bfb9c4b076afc562e4fb3d17c7db20 Mon Sep 17 00:00:00 2001 From: nico Date: Mon, 29 Jun 2020 18:59:16 +0200 Subject: flake8 fixup * fix trailing/ leading whitespaces * fix None comparison to use is + add noqa ignore statements for the influx escape replacements + add noqa ignore statements around the bare excepts TODO: This is only a temporary fix for most of the bare excepts, we need further specify those. --- README.md | 6 +++--- api.py | 2 +- cleanup.py | 6 +++--- influx.py | 4 ++-- prometheus.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7cbd2cf..ac33b00 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ listen: acl: api: - user: + user: - "api_user@magicbroccoli.de" loopback: ip: @@ -59,7 +59,7 @@ acl: api_permissions: "api access": from: - - mod_http_api + - mod_http_api who: access: allow: @@ -81,7 +81,7 @@ configuration file should be located at `/etc/ejabberd-metrics.yml`. #### systemd service To properly operate the metrics exporter tools, we created some systemd service templates, to simplify the whole process. If the `ejabberd-metrics.yml` file is not accessible for the user`nobody:nogroup`, it is required to update the -`User` and `Group` definitions inside the service file. +`User` and `Group` definitions inside the service file. If a virtualenv is used, it is required to update the `Environment=PATH` to include the `venv/bin` directory created earlier. diff --git a/api.py b/api.py index 3372976..6b83d2b 100644 --- a/api.py +++ b/api.py @@ -53,6 +53,6 @@ class EjabberdApi: return fn(self._login, data) return fn(data) - except: + except: # noqa: E722 # this needs to be more specific return {} diff --git a/cleanup.py b/cleanup.py index ee9def1..671c466 100755 --- a/cleanup.py +++ b/cleanup.py @@ -38,13 +38,13 @@ class EjabberdCleanup(EjabberdApiCalls): lastdate = None try: lastdate = datetime.datetime.strptime(last_stamp, "%Y-%m-%dT%H:%M:%SZ") - except: + except: # noqa: E722 try: lastdate = datetime.datetime.strptime(last_stamp, "%Y-%m-%dT%H:%M:%S.%fZ") - except: + except: # noqa: E722 logging.error(f"{user}@{host}: not able to parse '{last_stamp}'") return - if lastdate != None and lastdate - datetime.datetime.now() > datetime.timedelta( + if lastdate is not None and lastdate - datetime.datetime.now() > datetime.timedelta( days=self.offline_since_days ): self.delete_user(host, user, f"last seen @ {lastdate}") diff --git a/influx.py b/influx.py index b51d8f5..9437831 100644 --- a/influx.py +++ b/influx.py @@ -21,8 +21,8 @@ class Influx: @staticmethod def _rmspace(key: str = None, value: (str, int) = None): try: - key = key.replace(" ", "\ ") - value = value.replace(" ", "\ ") + key = key.replace(" ", "\ ") # noqa: W605 + value = value.replace(" ", "\ ") # noqa: W605 except (TypeError, AttributeError): pass diff --git a/prometheus.py b/prometheus.py index 6d7bbe5..a526851 100755 --- a/prometheus.py +++ b/prometheus.py @@ -23,7 +23,7 @@ class DynamicMetricsHandler(BaseHTTPRequestHandler): registry = registry.restricted_registry(params["name[]"]) try: output = generate_latest(registry) - except: + except: # noqa: E722 self.send_error(500, "error generating metric output") raise self.send_response(200) -- cgit v1.2.3-18-g5258