Permit empty/null active_visitors

README.org

@@ -1 +1,38 @@
 Inspired by [[https://twitter.com/rachel_shorey/status/1482097585447972866][this tweet.]]
+
+Monitor [[https://analytics.usa.gov][analytics.usa.gov]] for federal websites that currently have a larger
+number of unique visitors than the USPS package tracking system does.
+
+Display your results in the terminal every 10 minutes:
+[[[
+
+]]]
+- As JSON:
+  #+begin_src bash
+    user@host:~$fedmon --format json
+    {
+        "sites": [
+            {
+                "datetime": "2022-01-21T21:30:38.308841",
+                "active_visitors": 16748,
+                "page_title": "COVID Home Tests | USPS",
+                "page": "special.usps.com/testkits"
+            }
+        ]
+    }
+  #+end_src
+
+- As CSV
+  #+begin_src bash
+    user@host:~$fedmon --format csv
+    datetime,active_visitors,page_title,page
+    2022-01-21T21:30:32.720662,16748,COVID Home Tests | USPS,special.usps.com/testkits
+  #+end_src
+
+- Or as any format supported by [[https://github.com/astanin/python-tabulate#table-format][tabulate]]:
+  #+begin_src bash
+    user@host:~$fedmon --format orgtbl
+    | datetime                   |   active_visitors | page_title              | page                      |
+    |----------------------------+-------------------+-------------------------+---------------------------|
+    | 2022-01-21T21:12:04.025628 |             17714 | COVID Home Tests | USPS | special.usps.com/testkits |
+  #+end_src

fedmon/__init__.py

@@ -1 +1,2 @@
 __version__ = "0.1.0"
+appname = __name__

fedmon/cli.py

@@ -1,16 +1,28 @@
 import csv
 import functools
+import itertools
+import subprocess
 import time
-from typing import List
+from datetime import datetime
+from enum import Enum
+from io import StringIO
+from typing import List, Optional, Sequence, Tuple
 
 import cachetools.func
 import httpx
-import hyperlink
-import pendulum
 import typer
-from pydantic import BaseModel, BaseSettings
+from pydantic import BaseModel, BaseSettings, Field
+from tabulate import tabulate as tab
+from tabulate import tabulate_formats
+
+from fedmon import appname
 
 app = typer.Typer()
+tabulate_formats = set(tabulate_formats)
+
+AllowedFormat = Enum(
+    "AllowedFormat", {"json": "json", "csv": "csv", **{v: v for v in tabulate_formats}}
+)
 
 
 class AppConfig(BaseSettings):
@@ -26,7 +38,7 @@ config = AppConfig()
 
 
 @cachetools.func.ttl_cache(ttl=config.poll)
-def get_data(base_url: str):
+def get_data(base_url: str) -> Tuple[str]:
     with httpx.Client() as client:
         resp = client.get(base_url)
         resp.raise_for_status()
@@ -34,36 +46,108 @@ def get_data(base_url: str):
 
 
 class FedSite(BaseModel):
-    page: str
+    datetime: str = Field(default_factory=lambda: datetime.now().isoformat())
+    active_visitors: Optional[int]
     page_title: str
-    active_visitors: int
-
-    @property
-    def url(self):
-        return hyperlink.parse(self.page)
+    page: str
 
     class Config:
         frozen = True
 
 
+class Summary(BaseModel):
+    sites: Sequence[FedSite]
+
+
 @functools.lru_cache()
-def parse_data(data: List[str]):
+def parse_data(data: Tuple[str]) -> Tuple[FedSite]:
     return tuple(FedSite.parse_obj(line) for line in csv.DictReader(data))
 
 
 @functools.lru_cache()
-def analyze_data(data: List[FedSite]):
-    return data[0]
+def analyze_data(data: Tuple[FedSite], to_ignore: List[str]) -> Summary:
+    if to_ignore is None:
+        to_ignore = []
+    usps = {x for x in data if "tools.usps.com" in x.page}
+    usps_visitors = max(x.active_visitors for x in usps)
+    hot = tuple(
+        sorted(
+            (
+                x
+                for x in data
+                if x.active_visitors >= usps_visitors
+                and x not in usps
+                and not any(i in x.page for i in to_ignore)
+            ),
+            key=lambda x: x.active_visitors,
+        )
+    )
+    return Summary(sites=hot)
+
+
+def format_response(summary: Summary, format: AllowedFormat, count: int) -> str:
+    if format == AllowedFormat.json:
+        return summary.json(indent=4)
+    elif format == AllowedFormat.csv:
+        data = StringIO()
+        to_write = summary.dict()["sites"]
+        headers = FedSite.schema()["properties"].keys()
+        writer = csv.DictWriter(data, fieldnames=headers)
+        if count == 0:
+            writer.writeheader()
+        writer.writerows(to_write)
+        return data.getvalue()
+    else:
+        data = summary.dict()["sites"]
+        for site in data:
+            site["active_visitors"] = str(site["active_visitors"]).rjust(19)
+            site["page_title"] = site["page_title"].rjust(25)
+            site["page"] = site["page"].rjust(26)
+        if count == 0:
+            return tab(data, tablefmt=format.value, headers="keys")
+        else:
+            return tab(data, tablefmt=format.value)
+
+
+def notify(summary: Summary, counter) -> None:
+    try:
+        subprocess.check_call(
+            [
+                "notify-send",
+                appname,
+                format_response(summary, AllowedFormat.json, next(counter)),
+            ]
+        )
+    except subprocess.CalledProcessError as e:
+        typer.echo(e, err=True)
 
 
 @app.command()
-def main():
+def main(
+    format: AllowedFormat = typer.Option(
+        default="json", show_choices=True, help="The format to display results in."
+    ),
+    send_notification: bool = typer.Option(
+        default=False,
+        help="If set, use `notify-send` to raise a desktop notification, if available.",
+    ),
+    ignore: Optional[List[str]] = typer.Option(None),
+):
     prev = None
+    counter = itertools.count()
     while True:
-        data = get_data(config.base_url)
-        data = parse_data(data)
-        summary = analyze_data(data)
-        if summary != prev:
-            typer.echo(f"{pendulum.now()}: {summary}")
-            prev = summary
-        time.sleep(config.poll)
+        try:
+            data = get_data(config.base_url)
+            data = parse_data(data)
+            summary = analyze_data(data, to_ignore=ignore)
+            if summary != prev:
+                if summary.sites:
+                    c = next(counter)
+                    typer.echo(format_response(summary, format, c))
+                    if send_notification:
+                        notify(summary, c)
+                prev = summary
+        except (httpx.ConnectError, httpx.ReadTimeout) as e:
+            typer.echo(e, err=True)
+        finally:
+            time.sleep(config.poll)

poetry.lock

@@ -139,6 +139,14 @@ python-versions = ">=3.6"
 [package.dependencies]
 colorama = {version = "*", markers = "platform_system == \"Windows\""}
 
+[[package]]
+name = "cogapp"
+version = "3.3.0"
+description = "Cog: A content generator for executing Python snippets in source files."
+category = "dev"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
 [[package]]
 name = "colorama"
 version = "0.4.4"
@@ -229,17 +237,6 @@ brotli = ["brotlicffi", "brotli"]
 cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10.0.0,<11.0.0)", "pygments (>=2.0.0,<3.0.0)"]
 http2 = ["h2 (>=3,<5)"]
 
-[[package]]
-name = "hyperlink"
-version = "21.0.0"
-description = "A featureful, immutable, and correct URL for Python."
-category = "main"
-optional = false
-python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-
-[package.dependencies]
-idna = ">=2.5"
-
 [[package]]
 name = "identify"
 version = "2.4.4"
@@ -375,18 +372,6 @@ category = "dev"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
 
-[[package]]
-name = "pendulum"
-version = "2.1.2"
-description = "Python datetimes made easy"
-category = "main"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-
-[package.dependencies]
-python-dateutil = ">=2.6,<3.0"
-pytzdata = ">=2020.1"
-
 [[package]]
 name = "pexpect"
 version = "4.8.0"
@@ -539,25 +524,6 @@ wcwidth = "*"
 checkqa-mypy = ["mypy (v0.761)"]
 testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
 
-[[package]]
-name = "python-dateutil"
-version = "2.8.2"
-description = "Extensions to the standard Python datetime module"
-category = "main"
-optional = false
-python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
-
-[package.dependencies]
-six = ">=1.5"
-
-[[package]]
-name = "pytzdata"
-version = "2020.1"
-description = "The Olson timezone database for Python."
-category = "main"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-
 [[package]]
 name = "pyyaml"
 version = "6.0"
@@ -592,7 +558,7 @@ python-versions = "!=3.0,!=3.1,!=3.2,!=3.3,>=2.6"
 name = "six"
 version = "1.16.0"
 description = "Python 2 and 3 compatibility utilities"
-category = "main"
+category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 
@@ -620,6 +586,17 @@ pure-eval = "*"
 [package.extras]
 tests = ["pytest", "typeguard", "pygments", "littleutils"]
 
+[[package]]
+name = "tabulate"
+version = "0.8.9"
+description = "Pretty-print tabular data"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.extras]
+widechars = ["wcwidth"]
+
 [[package]]
 name = "toml"
 version = "0.10.2"
@@ -703,7 +680,7 @@ python-versions = "*"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "1d408cf93aa527fbbf46a14cbfd0311476ee0b96f2480372852328aa194ac59d"
+content-hash = "034532f84bbe75746626c4aa2a1a992b9a56b1189283dd6bff65fa7945f60ece"
 
 [metadata.files]
 anyio = [
@@ -754,6 +731,10 @@ click = [
     {file = "click-8.0.3-py3-none-any.whl", hash = "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3"},
     {file = "click-8.0.3.tar.gz", hash = "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"},
 ]
+cogapp = [
+    {file = "cogapp-3.3.0-py2.py3-none-any.whl", hash = "sha256:8b5b5f6063d8ee231961c05da010cb27c30876b2279e23ad0eae5f8f09460d50"},
+    {file = "cogapp-3.3.0.tar.gz", hash = "sha256:1be95183f70282422d594fa42426be6923070a4bd8335621f6347f3aeee81db0"},
+]
 colorama = [
     {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"},
 ]
@@ -785,10 +766,6 @@ httpx = [
     {file = "httpx-0.21.3-py3-none-any.whl", hash = "sha256:df9a0fd43fa79dbab411d83eb1ea6f7a525c96ad92e60c2d7f40388971b25777"},
     {file = "httpx-0.21.3.tar.gz", hash = "sha256:7a3eb67ef0b8abbd6d9402248ef2f84a76080fa1c839f8662e6eb385640e445a"},
 ]
-hyperlink = [
-    {file = "hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4"},
-    {file = "hyperlink-21.0.0.tar.gz", hash = "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b"},
-]
 identify = [
     {file = "identify-2.4.4-py2.py3-none-any.whl", hash = "sha256:aa68609c7454dbcaae60a01ff6b8df1de9b39fe6e50b1f6107ec81dcda624aa6"},
     {file = "identify-2.4.4.tar.gz", hash = "sha256:6b4b5031f69c48bf93a646b90de9b381c6b5f560df4cbe0ed3cf7650ae741e4d"},
@@ -833,29 +810,6 @@ pathspec = [
     {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"},
     {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"},
 ]
-pendulum = [
-    {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"},
-    {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"},
-    {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"},
-    {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"},
-    {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"},
-    {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"},
-    {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"},
-    {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"},
-    {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"},
-    {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"},
-    {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"},
-    {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"},
-    {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"},
-    {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"},
-    {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"},
-    {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"},
-    {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"},
-    {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"},
-    {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"},
-    {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"},
-    {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"},
-]
 pexpect = [
     {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"},
     {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"},
@@ -941,14 +895,6 @@ pytest = [
     {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
     {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"},
 ]
-python-dateutil = [
-    {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
-    {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
-]
-pytzdata = [
-    {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"},
-    {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"},
-]
 pyyaml = [
     {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"},
     {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"},
@@ -1004,6 +950,10 @@ stack-data = [
     {file = "stack_data-0.1.4-py3-none-any.whl", hash = "sha256:02cc0683cbc445ae4ca8c4e3a0e58cb1df59f252efb0aa016b34804a707cf9bc"},
     {file = "stack_data-0.1.4.tar.gz", hash = "sha256:7769ed2482ce0030e00175dd1bf4ef1e873603b6ab61cd3da443b410e64e9477"},
 ]
+tabulate = [
+    {file = "tabulate-0.8.9-py3-none-any.whl", hash = "sha256:d7c013fe7abbc5e491394e10fa845f8f32fe54f8dc60c6622c6cf482d25d47e4"},
+    {file = "tabulate-0.8.9.tar.gz", hash = "sha256:eb1d13f25760052e8931f2ef80aaf6045a6cceb47514db8beab24cded16f13a7"},
+]
 toml = [
     {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
     {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},

pyproject.toml

@@ -10,13 +10,13 @@ typer = {extras = ["all"], version = "^0.4.0"}
 cachetools = "^5.0.0"
 httpx = "^0.21.3"
 pydantic = "^1.9.0"
-hyperlink = "^21.0.0"
-pendulum = "^2.1.2"
+tabulate = "^0.8.9"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"
 ipython = "^8.0.1"
 pre-commit = "^2.17.0"
+cogapp = "^3.3.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]