grym
/
fedmon
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: grym:b924dd8a0b073510fff1fd56b4449a1429747a52
Branches Tags
grym:master
...
compare: grym:d55c14bbaa02f66cbd3204a9a1177f8f68efa406
Branches Tags
grym:master

2 Commits
b924dd8a0b ... d55c14bbaa

Author SHA1 Message Date
grym d55c14bbaa Update readme 2022-01-21 21:17:15 -05:00
grym 5c42951c70 Add support for csv, json, and tabulate-format output 2022-01-21 21:13:38 -05:00
4 changed files with 59 additions and 85 deletions
Show Stats Expand all files Collapse all files

6
README.org
View File

@ -1 +1,7 @@
Inspired by [[https://twitter.com/rachel_shorey/status/1482097585447972866][this tweet.]]
Monitor [[https://analytics.usa.gov][analytics.usa.gov]] for federal websites that currently have a larger
number of unique visitors than the USPS package tracking system does.
Display your results in the terminal every 10 minutes as JSON, CSV, or any
format supported by [[https://github.com/astanin/python-tabulate#table-format][tabulate]].

41
fedmon/cli.py
View File

@ -2,14 +2,23 @@ import csv
import functools
import time
from datetime import datetime
from enum import Enum
from io import StringIO
from typing import List, Tuple
import cachetools.func
import httpx
import tabulate
import typer
from pydantic import BaseModel, BaseSettings
from pydantic import BaseModel, BaseSettings, Field
from tabulate import tabulate as tab
app = typer.Typer()
tabulate_formats = set(tabulate.tabulate_formats)
AllowedFormat = Enum(
"AllowedFormat", {"json": "json", "csv": "csv", **{v: v for v in tabulate_formats}}
)
class AppConfig(BaseSettings):
@ -33,14 +42,19 @@ def get_data(base_url: str) -> Tuple[str]:
class FedSite(BaseModel):
page: str
page_title: str
datetime: str = Field(default_factory=lambda: datetime.now().isoformat())
active_visitors: int
page_title: str
page: str
class Config:
frozen = True
class Summary(BaseModel):
sites: Tuple[FedSite]
@functools.lru_cache()
def parse_data(data: Tuple[str]) -> Tuple[FedSite]:
return tuple(FedSite.parse_obj(line) for line in csv.DictReader(data))
@ -56,11 +70,26 @@ def analyze_data(data: Tuple[FedSite]) -> Tuple[FedSite]:
key=lambda x: x.active_visitors,
)
)
return hot
return Summary(sites=hot)
def format_response(summary: Summary, format: AllowedFormat) -> str:
if format == AllowedFormat.json:
return summary.json(indent=4)
elif format == AllowedFormat.csv:
data = StringIO()
to_write = summary.dict()["sites"]
headers = FedSite.schema()["properties"].keys()
writer = csv.DictWriter(data, fieldnames=headers)
writer.writeheader()
writer.writerows(summary.dict()["sites"])
return data.getvalue()
else:
return tab(summary.dict()["sites"], tablefmt=format.value, headers="keys")
@app.command()
def main():
def main(format: AllowedFormat = typer.Option(default="json", show_choices=True)):
prev = None
while True:
data = get_data(config.base_url)
@ -68,6 +97,6 @@ def main():
summary = analyze_data(data)
if summary != prev:
if summary:
typer.echo(f"{datetime.now().isoformat()}: {summary}")
typer.echo(format_response(summary, format))
prev = summary
time.sleep(config.poll)

96
poetry.lock generated
View File

@ -229,17 +229,6 @@ brotli = ["brotlicffi", "brotli"]
cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10.0.0,<11.0.0)", "pygments (>=2.0.0,<3.0.0)"]
http2 = ["h2 (>=3,<5)"]
[[package]]
name = "hyperlink"
version = "21.0.0"
description = "A featureful, immutable, and correct URL for Python."
category = "main"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[package.dependencies]
idna = ">=2.5"
[[package]]
name = "identify"
version = "2.4.4"
@ -375,18 +364,6 @@ category = "dev"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
[[package]]
name = "pendulum"
version = "2.1.2"
description = "Python datetimes made easy"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[package.dependencies]
python-dateutil = ">=2.6,<3.0"
pytzdata = ">=2020.1"
[[package]]
name = "pexpect"
version = "4.8.0"
@ -539,25 +516,6 @@ wcwidth = "*"
checkqa-mypy = ["mypy (v0.761)"]
testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
[[package]]
name = "python-dateutil"
version = "2.8.2"
description = "Extensions to the standard Python datetime module"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
[package.dependencies]
six = ">=1.5"
[[package]]
name = "pytzdata"
version = "2020.1"
description = "The Olson timezone database for Python."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "pyyaml"
version = "6.0"
@ -592,7 +550,7 @@ python-versions = "!=3.0,!=3.1,!=3.2,!=3.3,>=2.6"
name = "six"
version = "1.16.0"
description = "Python 2 and 3 compatibility utilities"
category = "main"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
@ -620,6 +578,17 @@ pure-eval = "*"
[package.extras]
tests = ["pytest", "typeguard", "pygments", "littleutils"]
[[package]]
name = "tabulate"
version = "0.8.9"
description = "Pretty-print tabular data"
category = "main"
optional = false
python-versions = "*"
[package.extras]
widechars = ["wcwidth"]
[[package]]
name = "toml"
version = "0.10.2"
@ -703,7 +672,7 @@ python-versions = "*"
[metadata]
lock-version = "1.1"
python-versions = "^3.8"
content-hash = "1d408cf93aa527fbbf46a14cbfd0311476ee0b96f2480372852328aa194ac59d"
content-hash = "176b6c7e99878be0ff9ee9f708a69835ff2b3eaa8fe2029427346d052453a694"
[metadata.files]
anyio = [
@ -785,10 +754,6 @@ httpx = [
{file = "httpx-0.21.3-py3-none-any.whl", hash = "sha256:df9a0fd43fa79dbab411d83eb1ea6f7a525c96ad92e60c2d7f40388971b25777"},
{file = "httpx-0.21.3.tar.gz", hash = "sha256:7a3eb67ef0b8abbd6d9402248ef2f84a76080fa1c839f8662e6eb385640e445a"},
]
hyperlink = [
{file = "hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4"},
{file = "hyperlink-21.0.0.tar.gz", hash = "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b"},
]
identify = [
{file = "identify-2.4.4-py2.py3-none-any.whl", hash = "sha256:aa68609c7454dbcaae60a01ff6b8df1de9b39fe6e50b1f6107ec81dcda624aa6"},
{file = "identify-2.4.4.tar.gz", hash = "sha256:6b4b5031f69c48bf93a646b90de9b381c6b5f560df4cbe0ed3cf7650ae741e4d"},
@ -833,29 +798,6 @@ pathspec = [
{file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"},
{file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"},
]
pendulum = [
{file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"},
{file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"},
{file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"},
{file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"},
{file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"},
{file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"},
{file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"},
{file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"},
{file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"},
{file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"},
{file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"},
{file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"},
{file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"},
{file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"},
{file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"},
{file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"},
{file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"},
{file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"},
{file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"},
{file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"},
{file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"},
]
pexpect = [
{file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"},
{file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"},
@ -941,14 +883,6 @@ pytest = [
{file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
{file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"},
]
python-dateutil = [
{file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
{file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
]
pytzdata = [
{file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"},
{file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"},
]
pyyaml = [
{file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"},
{file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"},
@ -1004,6 +938,10 @@ stack-data = [
{file = "stack_data-0.1.4-py3-none-any.whl", hash = "sha256:02cc0683cbc445ae4ca8c4e3a0e58cb1df59f252efb0aa016b34804a707cf9bc"},
{file = "stack_data-0.1.4.tar.gz", hash = "sha256:7769ed2482ce0030e00175dd1bf4ef1e873603b6ab61cd3da443b410e64e9477"},
]
tabulate = [
{file = "tabulate-0.8.9-py3-none-any.whl", hash = "sha256:d7c013fe7abbc5e491394e10fa845f8f32fe54f8dc60c6622c6cf482d25d47e4"},
{file = "tabulate-0.8.9.tar.gz", hash = "sha256:eb1d13f25760052e8931f2ef80aaf6045a6cceb47514db8beab24cded16f13a7"},
]
toml = [
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},

1
pyproject.toml
View File

@ -10,6 +10,7 @@ typer = {extras = ["all"], version = "^0.4.0"}
cachetools = "^5.0.0"
httpx = "^0.21.3"
pydantic = "^1.9.0"
tabulate = "^0.8.9"
[tool.poetry.dev-dependencies]
pytest = "^5.2"