From 6b7c55161f0d9c1e7e3546ccf001bb0eea43bf81 Mon Sep 17 00:00:00 2001 From: Lucidiot Date: Sun, 19 Jul 2020 16:43:02 +0200 Subject: [PATCH] Reorganize feed generation order Make agencies with less feeds generate first, to make debugging faster as those are the most likely to break --- feedgen.sh | 152 ++++++++++++++++++++++++++--------------------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/feedgen.sh b/feedgen.sh index 6893f61..01b73c8 100755 --- a/feedgen.sh +++ b/feedgen.sh @@ -40,6 +40,82 @@ curl -s 'https://www.taic.org.nz/inquiries?order=field_publication_date&sort=des | json2xml > $DIR/feeds/taic.xml.new \ && mv $DIR/feeds/taic.xml.new $DIR/feeds/taic.xml +log Building WMSC feed to $DIR/feeds/wmsc.xml +curl -s 'https://wmsc.gov/oversight/reports/' \ + | pup '.post-content li json{}' \ + | jq -f $DIR/jq/wmsc.jq \ + | json2xml > $DIR/feeds/wmsc.xml.new \ + && mv $DIR/feeds/wmsc.xml.new $DIR/feeds/wmsc.xml + +log Building DMAIB feed to $DIR/feeds/dmaib.xml +curl -s 'https://dmaib.com/reports/?categorizations=9510' \ + | pup 'ul.items > li[data-url] json{}' \ + | jq -f $DIR/jq/denmark.jq \ + --arg title 'DMAIB' \ + --arg description 'Danish Maritime Accident Investigation Board accident reports' \ + --arg link 'https://dmaib.com/reports/?categorizations=9510' \ + | json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/dmaib.xml.new \ + && mv $DIR/feeds/dmaib.xml.new $DIR/feeds/dmaib.xml + +log Building AIBD Aviation feed to $DIR/feeds/aibd/aviation.xml +mkdir -p $DIR/feeds/aibd +curl -s 'https://en.havarikommissionen.dk/aviation-archive/' \ + | pup 'ul.items > li[data-url] json{}' \ + | jq -f $DIR/jq/denmark.jq \ + --arg title 'AIBD Aviation' \ + --arg description 'Accident Investigation Board Denmark aviation accident reports' \ + --arg link 'https://en.havarikommissionen.dk/aviation-archive/' \ + | json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/aibd/aviation.xml.new \ + && mv $DIR/feeds/aibd/aviation.xml.new $DIR/feeds/aibd/aviation.xml + +log Building AIBD Rail feed to $DIR/feeds/aibd/rail.xml +curl -s 'https://en.havarikommissionen.dk/railway-archive/' \ + | pup 'ul.items > li[data-url] json{}' \ + | jq -f $DIR/jq/denmark.jq \ + --arg title 'AIBD Rail' \ + --arg description 'Accident Investigation Board Denmark railroad accident reports' \ + --arg link 'https://en.havarikommissionen.dk/railway-archive/' \ + | json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/aibd/rail.xml.new \ + && mv $DIR/feeds/aibd/rail.xml.new $DIR/feeds/aibd/rail.xml + +log Building BFU English feed to $DIR/feeds/bfu/en.xml +mkdir -p $DIR/feeds/bfu +curl -s 'https://www.bfu-web.de/EN/Publications/Investigation%20Report/reports_node.html?cms_gts=238148_list%253DdateOfIssue_dt%252Bdesc' \ + | pup 'table.links tbody tr json{}' \ + | jq -L $DIR/jq -f $DIR/jq/bfu/en.jq \ + | json2xml > $DIR/feeds/bfu/en.xml.new \ + && mv $DIR/feeds/bfu/en.xml.new $DIR/feeds/bfu/en.xml + +log Building BFU Deutsch feed to $DIR/feeds/bfu/de.xml +curl -s 'https://www.bfu-web.de/DE/Publikationen/Untersuchungsberichte/untersuchungsberichte_node.html?cms_gts=235998_list%253DdateOfIssue_dt%252Bdesc' \ + | pup 'table.links tbody tr json{}' \ + | jq -L $DIR/jq -f $DIR/jq/bfu/de.jq \ + | json2xml > $DIR/feeds/bfu/de.xml.new \ + && mv $DIR/feeds/bfu/de.xml.new $DIR/feeds/bfu/de.xml + +log Building UZPLN English feed to $DIR/feeds/uzpln/en.xml +mkdir -p $DIR/feeds/uzpln +curl -s 'https://www.uzpln.cz/en/reports' \ + | pup 'table.table tbody tr:not(:first-child) json{}' \ + | jq -L $DIR/jq \ + -f $DIR/jq/uzpln.jq \ + --arg language 'en' \ + --arg description 'Air Accidents Investigation Institute' \ + --arg link 'https://www.uzpln.cz/en/reports' \ + | json2xml > $DIR/feeds/uzpln/en.xml.new \ + && mv $DIR/feeds/uzpln/en.xml.new $DIR/feeds/uzpln/en.xml + +log Building UZPLN Czech feed to $DIR/feeds/uzpln/cz.xml +curl -s 'https://www.uzpln.cz/zpravy-ln' \ + | pup 'table.table tbody tr:not(:first-child) json{}' \ + | jq -L $DIR/jq \ + -f $DIR/jq/uzpln.jq \ + --arg language 'cz' \ + --arg description 'Ústav pro odborné Zjišťování Příčin Leteckých Nehod' \ + --arg link 'https://www.uzpln.cz/zpravy-ln' \ + | json2xml > $DIR/feeds/uzpln/cz.xml.new \ + && mv $DIR/feeds/uzpln/cz.xml.new $DIR/feeds/uzpln/cz.xml + log Building JTSB Aviation English feed to $DIR/feeds/jtsb/en/air.xml mkdir -p $DIR/feeds/jtsb/en curl -s 'https://www.mlit.go.jp/jtsb/airrep.html' \ @@ -84,82 +160,6 @@ curl -s 'https://jtsb.mlit.go.jp/jtsb/ship/ship-kensaku-list.php?sort=desc&by=p' | json2xml > $DIR/feeds/jtsb/jp/marine.xml.new \ && mv $DIR/feeds/jtsb/jp/marine.xml.new $DIR/feeds/jtsb/jp/marine.xml -log Building AIBD Aviation feed to $DIR/feeds/aibd/aviation.xml -mkdir -p $DIR/feeds/aibd -curl -s 'https://en.havarikommissionen.dk/aviation-archive/' \ - | pup 'ul.items > li[data-url] json{}' \ - | jq -f $DIR/jq/denmark.jq \ - --arg title 'AIBD Aviation' \ - --arg description 'Accident Investigation Board Denmark aviation accident reports' \ - --arg link 'https://en.havarikommissionen.dk/aviation-archive/' \ - | json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/aibd/aviation.xml.new \ - && mv $DIR/feeds/aibd/aviation.xml.new $DIR/feeds/aibd/aviation.xml - -log Building AIBD Rail feed to $DIR/feeds/aibd/rail.xml -curl -s 'https://en.havarikommissionen.dk/railway-archive/' \ - | pup 'ul.items > li[data-url] json{}' \ - | jq -f $DIR/jq/denmark.jq \ - --arg title 'AIBD Rail' \ - --arg description 'Accident Investigation Board Denmark railroad accident reports' \ - --arg link 'https://en.havarikommissionen.dk/railway-archive/' \ - | json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/aibd/rail.xml.new \ - && mv $DIR/feeds/aibd/rail.xml.new $DIR/feeds/aibd/rail.xml - -log Building DMAIB feed to $DIR/feeds/dmaib.xml -curl -s 'https://dmaib.com/reports/?categorizations=9510' \ - | pup 'ul.items > li[data-url] json{}' \ - | jq -f $DIR/jq/denmark.jq \ - --arg title 'DMAIB' \ - --arg description 'Danish Maritime Accident Investigation Board accident reports' \ - --arg link 'https://dmaib.com/reports/?categorizations=9510' \ - | json2xml -s -ns 'http://search.yahoo.com/mrss/' media > $DIR/feeds/dmaib.xml.new \ - && mv $DIR/feeds/dmaib.xml.new $DIR/feeds/dmaib.xml - -log Building BFU English feed to $DIR/feeds/bfu/en.xml -mkdir -p $DIR/feeds/bfu -curl -s 'https://www.bfu-web.de/EN/Publications/Investigation%20Report/reports_node.html?cms_gts=238148_list%253DdateOfIssue_dt%252Bdesc' \ - | pup 'table.links tbody tr json{}' \ - | jq -L $DIR/jq -f $DIR/jq/bfu/en.jq \ - | json2xml > $DIR/feeds/bfu/en.xml.new \ - && mv $DIR/feeds/bfu/en.xml.new $DIR/feeds/bfu/en.xml - -log Building BFU Deutsch feed to $DIR/feeds/bfu/de.xml -curl -s 'https://www.bfu-web.de/DE/Publikationen/Untersuchungsberichte/untersuchungsberichte_node.html?cms_gts=235998_list%253DdateOfIssue_dt%252Bdesc' \ - | pup 'table.links tbody tr json{}' \ - | jq -L $DIR/jq -f $DIR/jq/bfu/de.jq \ - | json2xml > $DIR/feeds/bfu/de.xml.new \ - && mv $DIR/feeds/bfu/de.xml.new $DIR/feeds/bfu/de.xml - -log Building UZPLN English feed to $DIR/feeds/uzpln/en.xml -mkdir -p $DIR/feeds/uzpln -curl -s 'https://www.uzpln.cz/en/reports' \ - | pup 'table.table tbody tr:not(:first-child) json{}' \ - | jq -L $DIR/jq \ - -f $DIR/jq/uzpln.jq \ - --arg language 'en' \ - --arg description 'Air Accidents Investigation Institute' \ - --arg link 'https://www.uzpln.cz/en/reports' \ - | json2xml > $DIR/feeds/uzpln/en.xml.new \ - && mv $DIR/feeds/uzpln/en.xml.new $DIR/feeds/uzpln/en.xml - -log Building UZPLN Czech feed to $DIR/feeds/uzpln/cz.xml -curl -s 'https://www.uzpln.cz/zpravy-ln' \ - | pup 'table.table tbody tr:not(:first-child) json{}' \ - | jq -L $DIR/jq \ - -f $DIR/jq/uzpln.jq \ - --arg language 'cz' \ - --arg description 'Ústav pro odborné Zjišťování Příčin Leteckých Nehod' \ - --arg link 'https://www.uzpln.cz/zpravy-ln' \ - | json2xml > $DIR/feeds/uzpln/cz.xml.new \ - && mv $DIR/feeds/uzpln/cz.xml.new $DIR/feeds/uzpln/cz.xml - -log Building WMSC feed to $DIR/feeds/wmsc.xml -curl -s 'https://wmsc.gov/oversight/reports/' \ - | pup '.post-content li json{}' \ - | jq -f $DIR/jq/wmsc.jq \ - | json2xml > $DIR/feeds/wmsc.xml.new \ - && mv $DIR/feeds/wmsc.xml.new $DIR/feeds/wmsc.xml - function saia_build { # saia_build language categoryId title description fileName log Building $3 feed to $DIR/feeds/saia/$1/$5