314 lines
16 KiB
Bash
Executable File
314 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
function usage {
|
|
echo "Usage: $0 [-d BASE_DIR] [-q]"
|
|
echo " -d, --base-dir Set the base directory for the feed generator files and output."
|
|
echo " -q, --quiet Hide status logs - useful when running the script as a cronjob."
|
|
exit 1
|
|
}
|
|
|
|
while [[ "$#" > 0 ]]; do case $1 in
|
|
-d|--base-dir) DIR="$2"; shift;;
|
|
-q|--quiet) QUIET=yes;;
|
|
*) echo "Unknown parameter: $1"; usage;;
|
|
esac; shift; done
|
|
|
|
function log {
|
|
test -z "$QUIET" && echo "$@"
|
|
}
|
|
|
|
[ -z "$DIR" ] && DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
|
log Building to $DIR/feeds
|
|
mkdir -p $DIR/feeds
|
|
|
|
if ! command -v json2xml >/dev/null 2>&1; then
|
|
log "Adding $DIR/bin to PATH"
|
|
PATH="$DIR/bin:$PATH"
|
|
fi
|
|
|
|
log Building ATSB feed to $DIR/feeds/atsb.xml
|
|
curl -s 'https://www.atsb.gov.au/publications/safety-investigation-reports/?s=1&sort=OccurrenceReleaseDate&sortAscending=descending&investigationStatus=Completed,Discontinued&printAll=true' \
|
|
| pup 'table.selectable_grid tr:not(.header) json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/atsb.jq \
|
|
| json2xml > $DIR/feeds/atsb.xml.new \
|
|
&& mv $DIR/feeds/atsb.xml.new $DIR/feeds/atsb.xml
|
|
|
|
log Building TAIC feed to $DIR/feeds/taic.xml
|
|
curl -s 'https://www.taic.org.nz/inquiries?order=field_publication_date&sort=desc' \
|
|
| pup '#view-table-wrapper tbody tr json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/taic.jq \
|
|
| json2xml > $DIR/feeds/taic.xml.new \
|
|
&& mv $DIR/feeds/taic.xml.new $DIR/feeds/taic.xml
|
|
|
|
log Building WMSC feed to $DIR/feeds/wmsc.xml
|
|
curl -s 'https://wmsc.gov/oversight/reports/' \
|
|
| pup '.post-content li json{}' \
|
|
| jq -f $DIR/jq/wmsc.jq \
|
|
| json2xml > $DIR/feeds/wmsc.xml.new \
|
|
&& mv $DIR/feeds/wmsc.xml.new $DIR/feeds/wmsc.xml
|
|
|
|
log Building DMAIB feed to $DIR/feeds/dmaib.xml
|
|
curl -s 'https://dmaib.com/reports/?categorizations=9510' \
|
|
| pup 'ul.items > li[data-url] json{}' \
|
|
| jq -f $DIR/jq/denmark.jq \
|
|
--arg title 'DMAIB' \
|
|
--arg description 'Danish Maritime Accident Investigation Board accident reports' \
|
|
--arg link 'https://dmaib.com/reports/?categorizations=9510' \
|
|
| json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/dmaib.xml.new \
|
|
&& mv $DIR/feeds/dmaib.xml.new $DIR/feeds/dmaib.xml
|
|
|
|
log Building SKYbrary feed to $DIR/feeds/skybrary.xml
|
|
curl -s 'https://www.skybrary.aero/api.php?action=askargs&conditions=Category:Accidents_and_Incidents&printouts=Date%20of%20Occurence|GeneralSub|Event%20Type|Aircraft¶meters=|sort%3DDate%20of%20Occurence|order%3Ddesc|limit%3D10000&format=json&api_version=3' \
|
|
| jq -f $DIR/jq/skybrary.jq \
|
|
| json2xml > $DIR/feeds/skybrary.xml.new \
|
|
&& mv $DIR/feeds/skybrary.xml.new $DIR/feeds/skybrary.xml
|
|
|
|
log Building AIBD Aviation feed to $DIR/feeds/aibd/aviation.xml
|
|
mkdir -p $DIR/feeds/aibd
|
|
curl -s 'https://en.havarikommissionen.dk/aviation-archive/' \
|
|
| pup 'ul.items > li[data-url] json{}' \
|
|
| jq -f $DIR/jq/denmark.jq \
|
|
--arg title 'AIBD Aviation' \
|
|
--arg description 'Accident Investigation Board Denmark aviation accident reports' \
|
|
--arg link 'https://en.havarikommissionen.dk/aviation-archive/' \
|
|
| json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/aibd/aviation.xml.new \
|
|
&& mv $DIR/feeds/aibd/aviation.xml.new $DIR/feeds/aibd/aviation.xml
|
|
|
|
log Building AIBD Rail feed to $DIR/feeds/aibd/rail.xml
|
|
curl -s 'https://en.havarikommissionen.dk/railway-archive/' \
|
|
| pup 'ul.items > li[data-url] json{}' \
|
|
| jq -f $DIR/jq/denmark.jq \
|
|
--arg title 'AIBD Rail' \
|
|
--arg description 'Accident Investigation Board Denmark railroad accident reports' \
|
|
--arg link 'https://en.havarikommissionen.dk/railway-archive/' \
|
|
| json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/aibd/rail.xml.new \
|
|
&& mv $DIR/feeds/aibd/rail.xml.new $DIR/feeds/aibd/rail.xml
|
|
|
|
log Building TTSB Aviation English feed to $DIR/feeds/ttsb/en/aviation.xml
|
|
mkdir -p $DIR/feeds/ttsb/en
|
|
curl -s 'https://www.ttsb.gov.tw/english/16051/16052/16053/16058/Lpsimplelist?PageSize=1000' \
|
|
| pup '#LP-ContentPage .TableList table tbody tr json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/ttsb.jq \
|
|
--arg title 'TTSB Aviation' \
|
|
--arg lang 'en-TW' \
|
|
--arg link 'https://www.ttsb.gov.tw/english/16051/16052/16053/16058/Lpsimplelist?PageSize=1000' \
|
|
| json2xml > $DIR/feeds/ttsb/en/aviation.xml.new \
|
|
&& mv $DIR/feeds/ttsb/en/aviation.xml.new $DIR/feeds/ttsb/en/aviation.xml
|
|
|
|
log Building TTSB Aviation Chinese feed to $DIR/feeds/ttsb/zh/aviation.xml
|
|
mkdir -p $DIR/feeds/ttsb/zh
|
|
curl -s 'https://www.ttsb.gov.tw/1133/1154/1155/1159/Lpsimplelist?PageSize=1000' \
|
|
| pup '#LP-ContentPage .TableList table tbody tr json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/ttsb.jq \
|
|
--arg title '國家運輸安全調查委員會' \
|
|
--arg lang 'zh-Hant-TW' \
|
|
--arg link 'https://www.ttsb.gov.tw/1133/1154/1155/1159/Lpsimplelist?PageSize=1000' \
|
|
| json2xml > $DIR/feeds/ttsb/zh/aviation.xml.new \
|
|
&& mv $DIR/feeds/ttsb/zh/aviation.xml.new $DIR/feeds/ttsb/zh/aviation.xml
|
|
|
|
log Building BFU English feed to $DIR/feeds/bfu/en.xml
|
|
mkdir -p $DIR/feeds/bfu
|
|
curl -s 'https://www.bfu-web.de/EN/Publications/Investigation%20Report/reports_node.html?cms_gts=238148_list%253DdateOfIssue_dt%252Bdesc' \
|
|
| pup 'table.links tbody tr json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/bfu/en.jq \
|
|
| json2xml > $DIR/feeds/bfu/en.xml.new \
|
|
&& mv $DIR/feeds/bfu/en.xml.new $DIR/feeds/bfu/en.xml
|
|
|
|
log Building BFU Deutsch feed to $DIR/feeds/bfu/de.xml
|
|
curl -s 'https://www.bfu-web.de/DE/Publikationen/Untersuchungsberichte/untersuchungsberichte_node.html?cms_gts=235998_list%253DdateOfIssue_dt%252Bdesc' \
|
|
| pup 'table.links tbody tr json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/bfu/de.jq \
|
|
| json2xml > $DIR/feeds/bfu/de.xml.new \
|
|
&& mv $DIR/feeds/bfu/de.xml.new $DIR/feeds/bfu/de.xml
|
|
|
|
log Building UZPLN English feed to $DIR/feeds/uzpln/en.xml
|
|
mkdir -p $DIR/feeds/uzpln
|
|
curl -s 'https://www.uzpln.cz/en/reports' \
|
|
| pup 'table.table tbody tr:not(:first-child) json{}' \
|
|
| jq -L $DIR/jq \
|
|
-f $DIR/jq/uzpln.jq \
|
|
--arg language 'en' \
|
|
--arg description 'Air Accidents Investigation Institute' \
|
|
--arg link 'https://www.uzpln.cz/en/reports' \
|
|
| json2xml > $DIR/feeds/uzpln/en.xml.new \
|
|
&& mv $DIR/feeds/uzpln/en.xml.new $DIR/feeds/uzpln/en.xml
|
|
|
|
log Building UZPLN Czech feed to $DIR/feeds/uzpln/cz.xml
|
|
curl -s 'https://www.uzpln.cz/zpravy-ln' \
|
|
| pup 'table.table tbody tr:not(:first-child) json{}' \
|
|
| jq -L $DIR/jq \
|
|
-f $DIR/jq/uzpln.jq \
|
|
--arg language 'cz' \
|
|
--arg description 'Ústav pro odborné Zjišťování Příčin Leteckých Nehod' \
|
|
--arg link 'https://www.uzpln.cz/zpravy-ln' \
|
|
| json2xml > $DIR/feeds/uzpln/cz.xml.new \
|
|
&& mv $DIR/feeds/uzpln/cz.xml.new $DIR/feeds/uzpln/cz.xml
|
|
|
|
log Building TSIB Marine feed to $DIR/feeds/tsib/marine.xml
|
|
mkdir -p $DIR/feeds/tsib
|
|
curl -s 'https://www.mot.gov.sg/about-mot/transport-safety-investigation-bureau/msib/investigation-report/' \
|
|
| pup '.container-twelve .eight > .row:last-child tbody > tr json{}' \
|
|
| jq -L $DIR/jq \
|
|
-f $DIR/jq/tsib.jq \
|
|
--arg type Marine \
|
|
--arg link 'https://www.mot.gov.sg/about-mot/transport-safety-investigation-bureau/msib/investigation-report/' \
|
|
| json2xml > $DIR/feeds/tsib/marine.xml.new \
|
|
&& mv $DIR/feeds/tsib/marine.xml.new $DIR/feeds/tsib/marine.xml
|
|
|
|
log Building TSIB Aviation feed to $DIR/feeds/tsib/air.xml
|
|
curl -s 'https://www.mot.gov.sg/about-mot/transport-safety-investigation-bureau/aaib/investigation-report/' \
|
|
| pup '.container-twelve .eight > .row:last-child tbody > tr json{}' \
|
|
| jq -L $DIR/jq \
|
|
-f $DIR/jq/tsib.jq \
|
|
--arg type Aviation \
|
|
--arg link 'https://www.mot.gov.sg/about-mot/transport-safety-investigation-bureau/aaib/investigation-report/' \
|
|
| json2xml > $DIR/feeds/tsib/air.xml.new \
|
|
&& mv $DIR/feeds/tsib/air.xml.new $DIR/feeds/tsib/air.xml
|
|
|
|
log Building JTSB Aviation English feed to $DIR/feeds/jtsb/en/air.xml
|
|
mkdir -p $DIR/feeds/jtsb/en
|
|
curl -s 'https://www.mlit.go.jp/jtsb/airrep.html' \
|
|
| pup 'table.kankokuiken-en tr:not(:first-child) json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/jtsb/en/air.jq \
|
|
| json2xml > $DIR/feeds/jtsb/en/air.xml.new \
|
|
&& mv $DIR/feeds/jtsb/en/air.xml.new $DIR/feeds/jtsb/en/air.xml
|
|
|
|
log Building JTSB Rail English feed to $DIR/feeds/jtsb/en/rail.xml
|
|
curl -s 'https://www.mlit.go.jp/jtsb/railrep.html' \
|
|
| pup 'table.kankokuiken-en tr:not(:first-child) json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/jtsb/en/rail.jq \
|
|
| json2xml > $DIR/feeds/jtsb/en/rail.xml.new \
|
|
&& mv $DIR/feeds/jtsb/en/rail.xml.new $DIR/feeds/jtsb/en/rail.xml
|
|
|
|
log Building JTSB Marine English feed to $DIR/feeds/jtsb/en/marine.xml
|
|
curl -s 'https://www.mlit.go.jp/jtsb/marrep.html' \
|
|
| pup 'table.kankokuiken-en tr:not(:first-child) json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/jtsb/en/marine.jq \
|
|
| json2xml > $DIR/feeds/jtsb/en/marine.xml.new \
|
|
&& mv $DIR/feeds/jtsb/en/marine.xml.new $DIR/feeds/jtsb/en/marine.xml
|
|
|
|
log Building JTSB Aviation Japanese feed to $DIR/feeds/jtsb/jp/air.xml
|
|
mkdir -p $DIR/feeds/jtsb/jp
|
|
curl -s 'https://jtsb.mlit.go.jp/jtsb/aircraft/air-kensaku-list.php?sort=desc&by=p' \
|
|
| pup 'table.jiko-information tr:not(:first-child) json{}' \
|
|
| jq -f $DIR/jq/jtsb/jp/air.jq \
|
|
| json2xml > $DIR/feeds/jtsb/jp/air.xml.new \
|
|
&& mv $DIR/feeds/jtsb/jp/air.xml.new $DIR/feeds/jtsb/jp/air.xml
|
|
|
|
log Building JTSB Rail Japanese feed to $DIR/feeds/jtsb/jp/rail.xml
|
|
curl -s 'https://jtsb.mlit.go.jp/jtsb/railway/rail-kensaku-list.php?sort=desc&by=p' \
|
|
| pup 'table.jiko-information tr:not(:first-child) json{}' \
|
|
| jq -f $DIR/jq/jtsb/jp/rail.jq \
|
|
| json2xml > $DIR/feeds/jtsb/jp/rail.xml.new \
|
|
&& mv $DIR/feeds/jtsb/jp/rail.xml.new $DIR/feeds/jtsb/jp/rail.xml
|
|
|
|
log Building JTSB Marine Japanese feed to $DIR/feeds/jtsb/jp/marine.xml
|
|
curl -s 'https://jtsb.mlit.go.jp/jtsb/ship/ship-kensaku-list.php?sort=desc&by=p' \
|
|
| pup 'table.jiko-information tr:not(:first-child) json{}' \
|
|
| jq -f $DIR/jq/jtsb/jp/marine.jq \
|
|
| json2xml > $DIR/feeds/jtsb/jp/marine.xml.new \
|
|
&& mv $DIR/feeds/jtsb/jp/marine.xml.new $DIR/feeds/jtsb/jp/marine.xml
|
|
|
|
function saia_build {
|
|
# saia_build language categoryId title description fileName
|
|
log Building $3 feed to $DIR/feeds/saia/$1/$5
|
|
mkdir -p $DIR/feeds/saia/$1
|
|
local url="https://www.havkom.se/$([ $1 == "en" ] && echo 'en/')utredningar?categoryId=$2&onGoingWithReport=1&sortReportDate=1"
|
|
curl -s "$url" \
|
|
| pup '.investigation .desc json{}' \
|
|
| jq -f $DIR/jq/saia.jq \
|
|
--arg title "$3" \
|
|
--arg language "$1" \
|
|
--arg description "$4" \
|
|
--arg link "$url" \
|
|
| json2xml > $DIR/feeds/saia/$1/$5.new \
|
|
&& mv $DIR/feeds/saia/$1/$5.new $DIR/feeds/saia/$1/$5
|
|
}
|
|
|
|
saia_build en 0 SAIA 'Swedish Accident Investigation Authority English accident reports' all.xml
|
|
saia_build en 203 'SAIA Road' 'Swedish Accident Investigation Authority English road accident reports' road.xml
|
|
saia_build en 204 'SAIA Rail' 'Swedish Accident Investigation Authority English rail accident reports' rail.xml
|
|
saia_build en 202 'SAIA Air' 'Swedish Accident Investigation Authority English aviation accident reports' air.xml
|
|
saia_build en 205 'SAIA Marine' 'Swedish Accident Investigation Authority English marine accident reports' marine.xml
|
|
saia_build en 206 'SAIA Military' 'Swedish Accident Investigation Authority English military accident reports' mil.xml
|
|
saia_build sv 0 SHK 'Svenska Statens haverikommission olycksrapporter' all.xml
|
|
saia_build sv 203 'SHK Väg' 'Svenska Statens haverikommission Vägtrafik olycksrapporter' road.xml
|
|
saia_build sv 204 'SHK Spärbunden' 'Svenska Statens haverikommission Spårbunden trafik olycksrapporter' rail.xml
|
|
saia_build sv 202 'SHK Luftfart' 'Svenska Statens haverikommission luftfart olycksrapporter' air.xml
|
|
saia_build sv 205 'SHK Sjöfart' 'Svenska Statens haverikommission sjöfart olycksrapporter' marine.xml
|
|
saia_build sv 206 'SHK Militär' 'Svenska Statens haverikommission militär olycksrapporter' mil.xml
|
|
|
|
log Building DAAI feed to $DIR/feeds/daai.xml
|
|
curl -sk -H 'User-Agent: Mozilla/1.0' https://mwt.gov.na/web/mwt/published-reports \
|
|
| pup 'table json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/daai.jq \
|
|
| json2xml > $DIR/feeds/daai.xml.new \
|
|
&& mv $DIR/feeds/daai.xml.new $DIR/feeds/daai.xml
|
|
|
|
log Building IoM Ship Registry feed to $DIR/feeds/iom.xml
|
|
curl -s https://www.iomshipregistry.com/forms-reports/casualty-reports/ \
|
|
| pup '.rte + .accordion p json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/iom.jq \
|
|
| json2xml > $DIR/feeds/iom.xml.new \
|
|
&& mv $DIR/feeds/iom.xml.new $DIR/feeds/iom.xml
|
|
|
|
log Building ZMA feed to $DIR/feeds/zma.xml
|
|
curl -s https://www.zma.go.tz/zma_casuality.php \
|
|
| pup '#content2 li a json{}' \
|
|
| jq -L $DIR/jq -f $DIR/jq/zma.jq \
|
|
| json2xml > $DIR/feeds/zma.xml.new \
|
|
&& mv $DIR/feeds/zma.xml.new $DIR/feeds/zma.xml
|
|
|
|
function taiid_build {
|
|
# taiid_build lang name description url fileName
|
|
log Building $2 feed to $DIR/feeds/taiid/$1/$5.xml
|
|
mkdir -p $DIR/feeds/taiid/$1
|
|
curl -s "$4" \
|
|
| pup '.panel-body tr:not(:first-child) json{}' \
|
|
| jq \
|
|
-L $DIR/jq \
|
|
-f $DIR/jq/taiid.jq \
|
|
--arg language "$1" \
|
|
--arg title "$2" \
|
|
--arg description "$3" \
|
|
--arg link "$4" \
|
|
| json2xml > $DIR/feeds/taiid/$1/$5.xml.new \
|
|
&& mv $DIR/feeds/taiid/$1/$5.xml.new $DIR/feeds/taiid/$1/$5.xml
|
|
}
|
|
|
|
taiid_build en TAIID 'Lithuania Transport Accident and Incident Investigations Division reports' 'https://tm.lrv.lt/en/fields-of-activity/transport-accident-and-incident-investigations/accidents-and-incidents' all
|
|
taiid_build en 'TAIID Aviation' 'Lithuania Transport Accident and Incident Investigations Division aviation reports' 'https://tm.lrv.lt/en/fields-of-activity/transport-accident-and-incident-investigations/civil-aviation' air
|
|
taiid_build en 'TAIID Rail' 'Lithuania Transport Accident and Incident Investigations Division rail reports' 'https://tm.lrv.lt/en/fields-of-activity/transport-accident-and-incident-investigations/railway-transport' rail
|
|
taiid_build en 'TAIID Maritime' 'Lithuania Transport Accident and Incident Investigations Division maritime reports' 'https://tm.lrv.lt/en/fields-of-activity/transport-accident-and-incident-investigations/maritime' marine
|
|
taiid_build lt TAITS 'Lietuvos Transporto Avarijų ir Incidentų Tyrimai Skyriaus ataskaitos' 'https://tm.lrv.lt/lt/veiklos-sritys-1/transporto-avariju-ir-incidentu-tyrimai-1/avarijos-ir-incidentai' all
|
|
taiid_build lt 'TAITS Aviacija' 'Lietuvos Transporto Avarijų ir Incidentų Tyrimai Skyriaus aviacija ataskaitos' 'https://tm.lrv.lt/lt/veiklos-sritys-1/transporto-avariju-ir-incidentu-tyrimai-1/oro-transportas' air
|
|
taiid_build lt 'TAITS Geležinkelio' 'Lietuvos Transporto Avarijų ir Incidentų Tyrimai Skyriaus geležinkelio ataskaitos' 'https://tm.lrv.lt/lt/veiklos-sritys-1/transporto-avariju-ir-incidentu-tyrimai-1/gelezinkeliu-transportas' rail
|
|
taiid_build lt 'TAITS Jūrinis' 'Lietuvos Transporto Avarijų ir Incidentų Tyrimai Skyriaus jūrinis ataskaitos' 'https://tm.lrv.lt/lt/veiklos-sritys-1/transporto-avariju-ir-incidentu-tyrimai-1/juru-transportas' marine
|
|
|
|
function sia_build {
|
|
# sia_build lang title description link
|
|
log Building $2 feed to $DIR/feeds/sia/$1.xml
|
|
mkdir -p $DIR/feeds/sia
|
|
curl -s 'https://sia.fi/api/oh-internet-sisallonhallinta/hae_nostolista.srv' \
|
|
-H 'Content-Type: application/json; charset=UTF-8' \
|
|
-d "{
|
|
\"listaus.sivustopolku\": \"/channels/public/www/otkes/$1/index/tutkintaselostukset\",
|
|
\"listaus.jsoninput\": \"{\\\"filterfields\\\":\\\"valmistunut_tutkinta\\\", \\\"valmistunut_tutkinta\\\":\\\"true\\\"}\"
|
|
}" \
|
|
| grep -v '^\s*$' \
|
|
| xml2json \
|
|
| jq \
|
|
-L $DIR/jq \
|
|
-f $DIR/jq/sia.jq \
|
|
--arg language "$1" \
|
|
--arg title "$2" \
|
|
--arg description "$3" \
|
|
--arg link "$4" \
|
|
| json2xml > $DIR/feeds/sia/$1.xml.new \
|
|
&& mv $DIR/feeds/sia/$1.xml.new $DIR/feeds/sia/$1.xml
|
|
}
|
|
|
|
sia_build en SIA 'Safety Investigation Authority reports' 'https://sia.fi/en/index/currentissues/completedinvestigations.html'
|
|
sia_build fi Onnettomuustutkintakeskus 'Onnettomuustutkintakeskus - Valmistuneet tutkinnat' 'https://sia.fi/fi/index/ajankohtaista/valmistuneettutkinnat.html'
|
|
sia_build sv Olycksutredningscentralen 'Olycksutredningscentralen - Publicerade rapporter' 'https://sia.fi/sv/index/aktuellt/publiceraderapporter.html'
|