From 75c3672e253b4b07a91ea4f86b314e1c31557498 Mon Sep 17 00:00:00 2001 From: "Buster \"Silver Eagle\" Neece" Date: Tue, 3 Jul 2018 17:51:05 -0500 Subject: [PATCH] #628 -- Switch all IP geolocation to be local via the MaxMind GeoLite DB. --- app/bootstrap/services.php | 7 +- app/locale/default.pot | 36 +++--- app/src/AzuraCast/Customization.php | 17 ++- .../AzuraCast/Middleware/GetCurrentUser.php | 11 +- app/src/Controller/Api/ApiProvider.php | 5 +- .../Controller/Api/ListenersController.php | 103 ++++++++---------- .../stations/reports/listeners.phtml | 5 +- composer.json | 3 +- composer.lock | 58 +++++++++- update.sh | 4 +- util/ansible/deploy.yml | 3 +- .../roles/azuracast-config/tasks/main.yml | 3 +- util/ansible/roles/maxmind/tasks/main.yml | 16 +++ util/ansible/update.yml | 3 +- 14 files changed, 179 insertions(+), 95 deletions(-) create mode 100644 util/ansible/roles/maxmind/tasks/main.yml diff --git a/app/bootstrap/services.php b/app/bootstrap/services.php index 24d90a98c..4962c8926 100644 --- a/app/bootstrap/services.php +++ b/app/bootstrap/services.php @@ -327,6 +327,11 @@ return function (\Slim\Container $di, $settings) { return \AzuraCast\Console\Application::create($di, $settings); }; + $di[MaxMind\Db\Reader::class] = function($di) { + $mmdb_path = dirname(APP_INCLUDE_ROOT).'/geoip/GeoLite2-City.mmdb'; + return new MaxMind\Db\Reader($mmdb_path); + }; + // // AzuraCast-specific dependencies // @@ -406,4 +411,4 @@ return function (\Slim\Container $di, $settings) { return $di; -}; \ No newline at end of file +}; diff --git a/app/locale/default.pot b/app/locale/default.pot index 6f6d2b57c..57332898c 100644 --- a/app/locale/default.pot +++ b/app/locale/default.pot @@ -7,8 +7,8 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"POT-Creation-Date: 2018-06-27T21:31:24+00:00\n" -"PO-Revision-Date: 2018-06-27T21:31:24+00:00\n" +"POT-Creation-Date: 2018-07-03T22:50:16+00:00\n" +"PO-Revision-Date: 2018-07-03T22:50:16+00:00\n" "Language: \n" #: /var/azuracast/www/app/config/admin/actions.conf.php:4 @@ -2819,8 +2819,8 @@ msgid "Title / File Path" msgstr "" #: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:21 -#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:92 -#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:195 +#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:95 +#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:198 msgid "Live Listeners" msgstr "" @@ -2856,27 +2856,31 @@ msgstr "" msgid "Unknown" msgstr "" -#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:196 -msgid "Today" -msgstr "" - -#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:197 -msgid "Yesterday" -msgstr "" - -#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:198 -msgid "Last 7 Days" +#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:82 +msgid "This product includes GeoLite2 data created by MaxMind, available from %s." msgstr "" #: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:199 -msgid "Last 30 Days" +msgid "Today" msgstr "" #: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:200 -msgid "This Month" +msgid "Yesterday" msgstr "" #: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:201 +msgid "Last 7 Days" +msgstr "" + +#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:202 +msgid "Last 30 Days" +msgstr "" + +#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:203 +msgid "This Month" +msgstr "" + +#: /var/azuracast/www/app/templates/stations/reports/listeners.phtml:204 msgid "Last Month" msgstr "" diff --git a/app/src/AzuraCast/Customization.php b/app/src/AzuraCast/Customization.php index 58c47bb63..c85301803 100644 --- a/app/src/AzuraCast/Customization.php +++ b/app/src/AzuraCast/Customization.php @@ -2,6 +2,7 @@ namespace AzuraCast; use App\Auth; +use App\Http\Request; use Entity; use Gettext\Translations; use Gettext\Translator; @@ -28,13 +29,18 @@ class Customization } /** - * Initialize timezone and locale settings for the current user. + * Initialize timezone and locale settings for the current user, and write them as attributes to the request. + * + * @param Request $request + * @return Request */ - public function init() + public function init(Request $request): Request { + $timezone = $this->getTimeZone(); + if (!APP_IS_COMMAND_LINE || APP_TESTING_MODE) { // Set time zone. - date_default_timezone_set($this->getTimeZone()); + date_default_timezone_set($timezone); // Localization $locale = $this->getLocale(); @@ -56,6 +62,11 @@ class Customization putenv("LANG=" . $locale); setlocale(LC_ALL, $locale); + + return $request->withAttributes([ + 'locale' => $locale, + 'timezone' => $timezone, + ]); } /** diff --git a/app/src/AzuraCast/Middleware/GetCurrentUser.php b/app/src/AzuraCast/Middleware/GetCurrentUser.php index 46be3053e..96fd8a1f1 100644 --- a/app/src/AzuraCast/Middleware/GetCurrentUser.php +++ b/app/src/AzuraCast/Middleware/GetCurrentUser.php @@ -2,12 +2,9 @@ namespace AzuraCast\Middleware; use App\Auth; -use AzuraCast\Assets; use AzuraCast\Customization; -use Doctrine\ORM\EntityManager; -use Slim\Container; -use Slim\Http\Request; -use Slim\Http\Response; +use App\Http\Request; +use App\Http\Response; /** * Get the current user entity object and assign it into the request if it exists. @@ -39,7 +36,7 @@ class GetCurrentUser // Initialize customization (timezones, locales, etc) based on the current logged in user. $this->customization->setUser($user); - $this->customization->init(); + $request = $this->customization->init($request); $request = $request ->withAttribute('user', $user) @@ -47,4 +44,4 @@ class GetCurrentUser return $next($request, $response); } -} \ No newline at end of file +} diff --git a/app/src/Controller/Api/ApiProvider.php b/app/src/Controller/Api/ApiProvider.php index 9aa7125f9..2d3fab924 100644 --- a/app/src/Controller/Api/ApiProvider.php +++ b/app/src/Controller/Api/ApiProvider.php @@ -26,7 +26,8 @@ class ApiProvider implements ServiceProviderInterface $di[ListenersController::class] = function($di) { return new ListenersController( $di[\Doctrine\ORM\EntityManager::class], - $di[\App\Cache::class] + $di[\App\Cache::class], + $di[\MaxMind\Db\Reader::class] ); }; @@ -67,4 +68,4 @@ class ApiProvider implements ServiceProviderInterface ); }; } -} \ No newline at end of file +} diff --git a/app/src/Controller/Api/ListenersController.php b/app/src/Controller/Api/ListenersController.php index 722eddb7e..39a0672e2 100644 --- a/app/src/Controller/Api/ListenersController.php +++ b/app/src/Controller/Api/ListenersController.php @@ -6,6 +6,7 @@ use Doctrine\ORM\EntityManager; use Entity; use App\Http\Request; use App\Http\Response; +use MaxMind\Db\Reader; class ListenersController { @@ -15,15 +16,20 @@ class ListenersController /** @var Cache */ protected $cache; + /** @var Reader */ + protected $geoip; + /** * ListenersController constructor. * @param EntityManager $em * @param Cache $cache + * @param Reader $geoip */ - public function __construct(EntityManager $em, Cache $cache) + public function __construct(EntityManager $em, Cache $cache, Reader $geoip) { $this->em = $em; $this->cache = $cache; + $this->geoip = $geoip; } /** @@ -84,14 +90,8 @@ class ListenersController ->getArrayResult(); } - $ips = []; - foreach($listeners_raw as $listener) { - $ips[$listener['listener_ip']] = $listener['listener_ip']; - } - - $ip_info = $this->_getIpInfo($ips); - $detect = new \Mobile_Detect; + $locale = $request->getAttribute('locale'); $listeners = []; foreach($listeners_raw as $listener) { @@ -103,7 +103,7 @@ class ListenersController $api->is_mobile = $detect->isMobile(); $api->connected_on = (int)$listener['timestamp_start']; $api->connected_time = $listener['connected_time'] ?? (time() - $listener['timestamp_start']); - $api->location = $ip_info[$listener['listener_ip']]; + $api->location = $this->_getLocationInfo($listener['listener_ip'], $locale); $listeners[] = $api; } @@ -111,56 +111,43 @@ class ListenersController return $response->withJson($listeners); } - protected function _getIpInfo($raw_ips) + protected function _getLocationInfo($ip, $locale): array { - $return = []; - foreach($raw_ips as $ip) { - $ip_info = $this->cache->get('/ip/'.$ip, null); - if ($ip_info !== null) { - $return[$ip] = $ip_info; - unset($raw_ips[$ip]); - } + $ip_info = $this->geoip->get($ip); + + if (empty($ip_info)) { + return [ + 'message' => 'Internal/Reserved IP', + ]; } - if (empty($raw_ips)) { - return $return; - } - - // Set up IP API batch query process. - $client = new \GuzzleHttp\Client([ - 'base_uri' => 'http://ip-api.com/batch', - 'timeout' => 10, - ]); - - $ips_per_request = 90; - - for($i = 0; $i <= count($raw_ips); $i += $ips_per_request) { - - $ips = array_slice($raw_ips, $i, $ips_per_request); - - $batch_json = []; - foreach($ips as $ip) { - $batch_json[] = ['query' => $ip]; - } - - $response = $client->post('', [ - 'json' => $batch_json, - ]); - - if ($response->getStatusCode() == 200) { - $response_body = $response->getBody()->getContents(); - $response = json_decode($response_body, true); - - foreach($response as $location_row) { - $ip = $location_row['query']; - unset($location_row['query']); - - $this->cache->set($location_row, '/ip/'.$ip, 3600); - $return[$ip] = $location_row; - } - } - } - - return $return; + return [ + 'region' => $this->_getLocalizedString($ip_info['subdivisions'][0]['names'] ?? null, $locale), + 'country' => $this->_getLocalizedString($ip_info['country']['names'] ?? null, $locale), + 'message' => 'This product includes GeoLite2 data created by MaxMind, available from http://www.maxmind.com.', + ]; } -} \ No newline at end of file + + protected function _getLocalizedString($names, $locale): string + { + if (empty($names)) { + return ''; + } + + // Convert "en_US" to "en-US", the format MaxMind uses. + $locale = str_replace('_', '-', $locale); + + // Check for an exact match. + if (isset($names[$locale])) { + return $names[$locale]; + } + + // Check for a match of the first portion, i.e. "en" + $locale = strtolower(substr($locale, 0, 2)); + if (isset($names[$locale])) { + return $names[$locale]; + } + + return $names['en']; + } +} diff --git a/app/templates/stations/reports/listeners.phtml b/app/templates/stations/reports/listeners.phtml index c87c00255..1840ab072 100644 --- a/app/templates/stations/reports/listeners.phtml +++ b/app/templates/stations/reports/listeners.phtml @@ -78,6 +78,9 @@ $assets +
+ http://www.maxmind.com') ?> +
@@ -218,4 +221,4 @@ $(function() { - \ No newline at end of file + diff --git a/composer.json b/composer.json index c25b2951a..b7860fb90 100644 --- a/composer.json +++ b/composer.json @@ -30,7 +30,8 @@ "monolog/monolog": "^1.23", "gettext/gettext": "^4.4", "cakephp/chronos": "^1.1", - "doctrine/data-fixtures": "^1.3" + "doctrine/data-fixtures": "^1.3", + "maxmind-db/reader": "~1.0" }, "require-dev": { "codeception/codeception": "^2.2", diff --git a/composer.lock b/composer.lock index 609281c50..5954f1263 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "feeae2a7fb9556b1199f25d1ca393320", + "content-hash": "f4a8f439fed1855416ed1d5f27ae6bdf", "packages": [ { "name": "azuracast/azuraforms", @@ -1686,6 +1686,62 @@ ], "time": "2017-02-19T11:47:49+00:00" }, + { + "name": "maxmind-db/reader", + "version": "v1.3.0", + "source": { + "type": "git", + "url": "https://github.com/maxmind/MaxMind-DB-Reader-php.git", + "reference": "e042b4f8a2dff41e19019faf16427178b07fbd58" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/maxmind/MaxMind-DB-Reader-php/zipball/e042b4f8a2dff41e19019faf16427178b07fbd58", + "reference": "e042b4f8a2dff41e19019faf16427178b07fbd58", + "shasum": "" + }, + "require": { + "php": ">=5.4" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "2.*", + "phpunit/phpunit": "4.* || 5.*", + "satooshi/php-coveralls": "1.0.*", + "squizlabs/php_codesniffer": "3.*" + }, + "suggest": { + "ext-bcmath": "bcmath or gmp is required for decoding larger integers with the pure PHP decoder", + "ext-gmp": "bcmath or gmp is required for decoding larger integers with the pure PHP decoder", + "ext-maxminddb": "A C-based database decoder that provides significantly faster lookups" + }, + "type": "library", + "autoload": { + "psr-4": { + "MaxMind\\Db\\": "src/MaxMind/Db" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Gregory J. Oschwald", + "email": "goschwald@maxmind.com", + "homepage": "http://www.maxmind.com/" + } + ], + "description": "MaxMind DB Reader API", + "homepage": "https://github.com/maxmind/MaxMind-DB-Reader-php", + "keywords": [ + "database", + "geoip", + "geoip2", + "geolocation", + "maxmind" + ], + "time": "2018-02-21T21:23:33+00:00" + }, { "name": "mobiledetect/mobiledetectlib", "version": "2.8.31", diff --git a/update.sh b/update.sh index 511c65dd6..6fca01d1c 100755 --- a/update.sh +++ b/update.sh @@ -33,7 +33,7 @@ else fi APP_ENV="${APP_ENV:-production}" -UPDATE_REVISION="${UPDATE_REVISION:-23}" +UPDATE_REVISION="${UPDATE_REVISION:-24}" echo "Updating AzuraCast (Environment: $APP_ENV, Update revision: $UPDATE_REVISION)" @@ -41,4 +41,4 @@ if [ $APP_ENV = "production" ]; then git reset --hard && git pull fi -ansible-playbook util/ansible/update.yml --inventory=util/ansible/hosts --extra-vars "app_env=$APP_ENV update_revision=$UPDATE_REVISION" \ No newline at end of file +ansible-playbook util/ansible/update.yml --inventory=util/ansible/hosts --extra-vars "app_env=$APP_ENV update_revision=$UPDATE_REVISION" diff --git a/util/ansible/deploy.yml b/util/ansible/deploy.yml index 7390323a0..7e8fe476d 100644 --- a/util/ansible/deploy.yml +++ b/util/ansible/deploy.yml @@ -24,8 +24,9 @@ - mariadb - azuracast-db-install - ufw + - maxmind - composer - influxdb - services - azuracast-setup - - azuracast-cron \ No newline at end of file + - azuracast-cron diff --git a/util/ansible/roles/azuracast-config/tasks/main.yml b/util/ansible/roles/azuracast-config/tasks/main.yml index 919e95ded..0ee47332a 100644 --- a/util/ansible/roles/azuracast-config/tasks/main.yml +++ b/util/ansible/roles/azuracast-config/tasks/main.yml @@ -20,6 +20,7 @@ - "{{ tmp_base }}" - "{{ tmp_base }}/proxies" - "{{ app_base }}/stations" + - "{{ app_base }}/geoip" - "{{ app_base }}/servers" - "{{ app_base }}/servers/shoutcast2" - - "{{ app_base }}/servers/icecast2" \ No newline at end of file + - "{{ app_base }}/servers/icecast2" diff --git a/util/ansible/roles/maxmind/tasks/main.yml b/util/ansible/roles/maxmind/tasks/main.yml new file mode 100644 index 000000000..dba46f0ba --- /dev/null +++ b/util/ansible/roles/maxmind/tasks/main.yml @@ -0,0 +1,16 @@ +--- + - name: Download MaxMind GeoIP Database + get_url: + url: http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz + dest: "{{ app_base }}/geoip/maxmind.tar.gz" + + - name: Extract MaxMind GeoIP Database + unarchive: + src: "{{ app_base }}/geoip/maxmind.tar.gz" + dest: "{{ app_base }}/geoip" + remote_src: yes + creates: "{{ app_base }}/geoip/GeoLite2-City.mmdb" + mode: "u=rwx,g=rx,o=rx" + owner: "azuracast" + group: "www-data" + extra_opts: "--strip-components=1" diff --git a/util/ansible/update.yml b/util/ansible/update.yml index 2a29ed048..831d0ad3a 100644 --- a/util/ansible/update.yml +++ b/util/ansible/update.yml @@ -22,6 +22,7 @@ - composer - { role: influxdb, when: update_revision|int < 10 } - { role: ufw, when: update_revision|int < 12 } + - { role: maxmind, when: update_revision|int < 24 } - { role: services, when: update_revision|int < 13 } - { role: azuracast-cron, when: update_revision|int < 2 } - - azuracast-setup \ No newline at end of file + - azuracast-setup