lucidiot/itsb
lucidiot
/
itsb
1
1
Fork 0
Code Issues 61 Pull Requests Releases Wiki Activity

Compare commits

base: lucidiot:fc7f292007c0e081497a87efad64cf12c6dfeaa3
Branches Tags
lucidiot:main
...
compare: lucidiot:aba2b321d8d06ae285e1fe4809b7907f2c2c0b76
Branches Tags
lucidiot:main

3 Commits
fc7f292007 ... aba2b321d8

Author SHA1 Message Date
~lucidiot aba2b321d8
Call new GoBasic API for Danish reports
continuous-integration/drone Build is passing Details
2023-05-09 11:52:29 +02:00
~lucidiot 6b1950309c
Fix BEA custom feed 2023-05-09 10:13:58 +02:00
~lucidiot 64ae2ee948
Ignore expired SSL cert on BEA-TT 2023-05-09 09:15:54 +02:00
5 changed files with 79 additions and 48 deletions
Show Stats Expand all files Collapse all files

62
itsb.xml
View File

@ -64,7 +64,7 @@
<frequency>10-20 reports/year</frequency> <frequency>10-20 reports/year</frequency>
<url>https://www.bea-tt.developpement-durable.gouv.fr/</url> <url>https://www.bea-tt.developpement-durable.gouv.fr/</url>
<feed lang="French" format="rss" id="beatt-rss"> <feed lang="French" format="rss" id="beatt-rss">
<link>https://www.bea-tt.developpement-durable.gouv.fr/spip.php?page=backend-actu</link> <link verify-ssl="false">https://www.bea-tt.developpement-durable.gouv.fr/spip.php?page=backend-actu</link>
</feed> </feed>
</source> </source>
@ -338,11 +338,9 @@
<curl> <curl>
<url verify-ssl="false">https://bea.aero/les-enquetes/evenements-notifies/?tx_news_pi1%5Baction%5D=searchResult&amp;tx_news_pi1%5Bcontroller%5D=News&amp;tx_news_pi1%5BfacetAction%5D=add&amp;tx_news_pi1%5BfacetTitle%5D=etat_intS&amp;tx_news_pi1%5BfacetValue%5D=100&amp;cHash=b4645da2a30ec0c83f60069d018b5619</url> <url verify-ssl="false">https://bea.aero/les-enquetes/evenements-notifies/?tx_news_pi1%5Baction%5D=searchResult&amp;tx_news_pi1%5Bcontroller%5D=News&amp;tx_news_pi1%5BfacetAction%5D=add&amp;tx_news_pi1%5BfacetTitle%5D=etat_intS&amp;tx_news_pi1%5BfacetValue%5D=100&amp;cHash=b4645da2a30ec0c83f60069d018b5619</url>
</curl> </curl>
<pup>article.search-entry</pup> <pup>.search-entry__event</pup>
<jq path="bea.jq" /> <jq path="bea.jq" />
<json2xml> <json2xml />
<namespace prefix="georss">http://www.georss.org/georss</namespace>
</json2xml>
<output>bea.xml</output> <output>bea.xml</output>
</feed> </feed>
</source> </source>
@ -417,13 +415,23 @@
<url>https://havarikommissionen.dk/</url> <url>https://havarikommissionen.dk/</url>
<feed format="rss" lang="Danish, English" type="aviation" id="aibd-aviation"> <feed format="rss" lang="Danish, English" type="aviation" id="aibd-aviation">
<curl> <curl>
<url>https://en.havarikommissionen.dk/aviation-archive/</url> <url>https://en.havarikommissionen.dk/investigation-results/search-aviation/</url>
</curl> </curl>
<pup>ul.items > li[data-url]</pup> <pup output="text">.archive-search-result + script</pup>
<jq path="denmark.jq"> <shell><![CDATA[grep -o '{.*}']]></shell>
<jq path="denmark/query.jq">
<arg name="category"></arg>
</jq>
<curl>
<url>https://en.havarikommissionen.dk/proxy.gba</url>
<requestBody>@-</requestBody>
</curl>
<jq path="denmark/extract.jq" raw-output="true" />
<pup>.item[data-url]</pup>
<jq path="denmark/build.jq">
<arg name="title">AIBD Aviation</arg> <arg name="title">AIBD Aviation</arg>
<arg name="description">Accident Investigation Board Denmark aviation accident reports</arg> <arg name="description">Accident Investigation Board Denmark aviation accident reports</arg>
<arg name="link">https://en.havarikommissionen.dk/aviation-archive/</arg> <arg name="link">https://en.havarikommissionen.dk/investigation-results/search-aviation/</arg>
</jq> </jq>
<json2xml short-empty="true"> <json2xml short-empty="true">
<namespace prefix="media">http://search.yahoo.com/mrss/</namespace> <namespace prefix="media">http://search.yahoo.com/mrss/</namespace>
@ -432,13 +440,23 @@
</feed> </feed>
<feed format="rss" lang="Danish, English" type="rail" id="aibd-rail"> <feed format="rss" lang="Danish, English" type="rail" id="aibd-rail">
<curl> <curl>
<url>https://en.havarikommissionen.dk/railway-archive/</url> <url>https://en.havarikommissionen.dk/investigation-results/search-railway/</url>
</curl> </curl>
<pup>ul.items > li[data-url]</pup> <pup output="text">.archive-search-result + script</pup>
<jq path="denmark.jq"> <shell><![CDATA[grep -o '{.*}']]></shell>
<jq path="denmark/query.jq">
<arg name="category"></arg>
</jq>
<curl>
<url>https://en.havarikommissionen.dk/proxy.gba</url>
<requestBody>@-</requestBody>
</curl>
<jq path="denmark/extract.jq" raw-output="true" />
<pup>.item[data-url]</pup>
<jq path="denmark/build.jq">
<arg name="title">AIBD Rail</arg> <arg name="title">AIBD Rail</arg>
<arg name="description">Accident Investigation Board Denmark railroad accident reports</arg> <arg name="description">Accident Investigation Board Denmark railroad accident reports</arg>
<arg name="link">https://en.havarikommissionen.dk/railway-archive/</arg> <arg name="link">https://en.havarikommissionen.dk/investigation-results/search-railway/</arg>
</jq> </jq>
<json2xml short-empty="true"> <json2xml short-empty="true">
<namespace prefix="media">http://search.yahoo.com/mrss/</namespace> <namespace prefix="media">http://search.yahoo.com/mrss/</namespace>
@ -455,13 +473,23 @@
<url>https://dmaib.com/</url> <url>https://dmaib.com/</url>
<feed format="rss" lang="Danish, English" id="dmaib-rss"> <feed format="rss" lang="Danish, English" id="dmaib-rss">
<curl> <curl>
<url>https://dmaib.com/reports/?categorizations=9510</url> <url>https://dmaib.com/reports/</url>
</curl> </curl>
<pup>ul.items > li[data-url]</pup> <pup output="text">.archive-search-result + script</pup>
<jq path="denmark.jq"> <shell><![CDATA[grep -o '{.*}']]></shell>
<jq path="denmark/query.jq">
<arg name="category">8384</arg>
</jq>
<curl>
<url>https://dmaib.com/proxy.gba</url>
<requestBody>@-</requestBody>
</curl>
<jq path="denmark/extract.jq" raw-output="true" />
<pup>.item[data-url]</pup>
<jq path="denmark/build.jq">
<arg name="title">DMAIB</arg> <arg name="title">DMAIB</arg>
<arg name="description">Danish Maritime Accident Investigation Board accident reports</arg> <arg name="description">Danish Maritime Accident Investigation Board accident reports</arg>
<arg name="link">https://dmaib.com/reports/?categorizations=9510</arg> <arg name="link">https://dmaib.com/reports/</arg>
</jq> </jq>
<json2xml short-empty="true"> <json2xml short-empty="true">
<namespace prefix="media">http://search.yahoo.com/mrss/</namespace> <namespace prefix="media">http://search.yahoo.com/mrss/</namespace>

32
jq/bea.jq
View File

@ -1,14 +1,11 @@
# BEA feed generator # BEA feed generator
# Expects pup JSON output holding <article> tags, outputs xmltodict-compatible JSON # Expects pup JSON output holding <div class="search-entry__event"> tags, outputs xmltodict-compatible JSON
# WARNING: Dates are locale-sensitive; the RSS feed might not generate correctly with another locale. # WARNING: Dates are locale-sensitive; the RSS feed might not generate correctly with another locale.
import "./helpers" as helpers; import "./helpers" as helpers;
{ {
"rss": { "rss": {
"@version": "2.0", "@version": "2.0",
"@xmlns": {
"georss": "http://www.georss.org/georss"
},
"channel": { "channel": {
"title": "BEA", "title": "BEA",
"description": "Rapports d'enquête publiés par le Bureau d'Enquêtes et Analyses", "description": "Rapports d'enquête publiés par le Bureau d'Enquêtes et Analyses",
@ -19,25 +16,20 @@ import "./helpers" as helpers;
"ttl": 1440, "ttl": 1440,
"generator": "ITSB", "generator": "ITSB",
"item": [.[] | { "item": [.[] | {
"title": (.children[1].children[1].title | gsub("&#39;"; "'")), "title": (.children[0].children[-1].children[0].title | gsub("&#39;"; "'")),
"description": ( "description": (
[ "<p>" + .children[0].children[-1].children[1].text + "</p>" + (
.children[0].children[1].children[].children [
| .[0].text + " : " + ((.[1].children[0] | .text // .children[0].text) // .[1].text) .children[1].children[].children
| "<dt>\(.[0].text)</dt><dd>\(.[1].children[0].text)</dd>"
]
| join("")
| if length then "<dl>\(.)</dl>" else "" end
| gsub("&#39;"; "'") | gsub("&#39;"; "'")
] )
| join("</li><li>")
| "<ul><li>\(.)</li></ul>"
), ),
"link": (.children[1].children[1].href | helpers::urlresolve("https://bea.aero/")), "link": (.children[0].children[-1].children[0].href | helpers::urlresolve("https://bea.aero/")),
"guid": (.children[1].children[1].href | helpers::urlresolve("https://bea.aero/")), "guid": (.children[0].children[-1].children[0].href | helpers::urlresolve("https://bea.aero/"))
"pubDate": (
[.children[0].children[1].children[].children[].children[]?.children[0].text | strings][0]
| strptime("%d/%m/%Y")
| mktime
| strftime("%a, %d %b %Y %T %z")
),
"http://www.georss.org/georss:point": (.children[2].text | [match("', (-?[0-9.]+), (-?[0-9.]+)") | .captures[].string] | join(" "))
}] }]
} }
} }

20
jq/denmark.jq → jq/denmark/build.jq
View File

@ -1,12 +1,9 @@
# Denmark feed generator # Accident Investigation Board Denmark and Danish Maritime Accident Investigation Board feed generator
# Compatible with AIBD Aviation https://en.havarikommissionen.dk/aviation-archive/
# Compatible with AIBD Rail https://en.havarikommissionen.dk/railway-archive/
# Compatible with DMAIB https://dmaib.com/reports/?categorizations=9510
# Required arguments: # Required arguments:
# $title: Channel title # $title: Channel title
# $description: Channel description # $description: Channel description
# $link: Channel link # $link: Channel link
# Expects pup JSON output holding <li> tags, outputs xmltodict-compatible JSON # Expects pup JSON output holding <div class="item"> tags, outputs xmltodict-compatible JSON
# WARNING: Dates are locale-sensitive; the RSS feed might not generate correctly with another locale. # WARNING: Dates are locale-sensitive; the RSS feed might not generate correctly with another locale.
import "./helpers" as helpers; import "./helpers" as helpers;
@ -20,7 +17,7 @@ import "./helpers" as helpers;
"title": $title, "title": $title,
"description": $description, "description": $description,
"link": $link, "link": $link,
"language": "en", "language": "en-DK",
"pubDate": (now | strftime("%a, %d %b %Y %T %z")), "pubDate": (now | strftime("%a, %d %b %Y %T %z")),
"docs": "https://www.rssboard.org/rss-specification", "docs": "https://www.rssboard.org/rss-specification",
"ttl": 1440, "ttl": 1440,
@ -29,27 +26,28 @@ import "./helpers" as helpers;
.[] .[]
| . as $item | . as $item
| { | {
"title": .children[-1].children[0].children[0].text, "title": .children[-1].children[-4].children[0].text,
"description": .children[-1].children[-1].text, "description": .children[-1].children[-1].text,
"link": (.["data-url"] | helpers::urlresolve($link)), "link": (.["data-url"] | helpers::urlresolve($link)),
"guid": (.["data-url"] | helpers::urlresolve($link)), "guid": (.["data-url"] | helpers::urlresolve($link)),
"pubDate": ( "pubDate": (
.children[-1].children[1].text[-10:] .children[-1].children[-3].text
| if startswith("Published ") then .[-10:] else . end
| strptime("%d-%m-%Y") | strptime("%d-%m-%Y")
| mktime | mktime
| strftime("%a, %d %b %Y %T %z") | strftime("%a, %d %b %Y %T %z")
), ),
"category": [ "category": [
.children[-1].children[2].children[].text | { .children[-1].children[-2].children[].children[0].text | {
"@domain": $link, "@domain": $link,
"#text": . "#text": .
} }
] ]
} }
# Optional MediaRSS thumbnail # Optional MediaRSS thumbnail
| if $item.children[0].class == "media-left" then | if $item.children[0].class == "media" then
.["http://search.yahoo.com/mrss/:thumbnail"] = { .["http://search.yahoo.com/mrss/:thumbnail"] = {
"@url": ($item.children[0].children[0].src | helpers::urlresolve($link)) "@url": ($item.children[0].children[0].children[-1].src | helpers::urlresolve($link))
} }
else . end else . end
] ]

1
jq/denmark/extract.jq Normal file
View File

@ -0,0 +1 @@
.value.page

12
jq/denmark/query.jq Normal file
View File

@ -0,0 +1,12 @@
{
"control": "GoBasic.Presentation.Controls.ListHelper, GoBasic.Presentation",
"method": "GetPage",
"path": "",
"query": "",
"args": {
"arg0": .,
"arg1": 1,
"arg2": {"categorizations": ([$category] | map(select(length > 0)))},
"arg3": "",
}
}