From 04bd45fc2d37949608571f54f75a173b1d471644 Mon Sep 17 00:00:00 2001 From: Vaalyn Date: Tue, 22 Mar 2022 05:35:35 +0100 Subject: [PATCH] Update report exports to use league csv (#5132) --- .../scss/vendors/_daterangepicker-colors.scss | 10 ++ .../components/Stations/Reports/Listeners.vue | 12 +- .../components/Stations/Reports/Timeline.vue | 19 ++- .../Api/Stations/HistoryController.php | 125 +++++++++------- .../Api/Stations/ListenersAction.php | 40 +++--- .../Stations/Reports/PerformanceAction.php | 109 ++++++++------ src/Utilities/Csv.php | 43 ------ src/Utilities/Xml.php | 99 ------------- tests/Functional/Api_Stations_ReportsCest.php | 136 ++++++++++++++++++ tests/Unit/ExportsTest.php | 34 ----- tests/Unit/XmlTest.php | 6 +- 11 files changed, 325 insertions(+), 308 deletions(-) delete mode 100644 src/Utilities/Csv.php delete mode 100644 src/Utilities/Xml.php delete mode 100644 tests/Unit/ExportsTest.php diff --git a/frontend/scss/vendors/_daterangepicker-colors.scss b/frontend/scss/vendors/_daterangepicker-colors.scss index 646089447..b98dcb457 100644 --- a/frontend/scss/vendors/_daterangepicker-colors.scss +++ b/frontend/scss/vendors/_daterangepicker-colors.scss @@ -25,6 +25,10 @@ .calendar-table { background: none; border: 0; + + .next span, .prev span { + border-color: $text-color; + } } td.off { @@ -39,4 +43,10 @@ th.available:hover { background-color: $navdrawer-nav-link-bg-hover; } + + .monthselect, .yearselect, input, select.ampmselect, select.hourselect, select.minuteselect, select.secondselect { + background-color: $menu-bg; + border: 1px solid $menu-divider-bg; + color: $text-color; + } } diff --git a/frontend/vue/components/Stations/Reports/Listeners.vue b/frontend/vue/components/Stations/Reports/Listeners.vue index 7a4a10537..0452a5016 100644 --- a/frontend/vue/components/Stations/Reports/Listeners.vue +++ b/frontend/vue/components/Stations/Reports/Listeners.vue @@ -202,16 +202,16 @@ export default { return DateTime.fromJSDate(this.liveTime).equals(DateTime.fromJSDate(this.dateRange.startDate)); }, exportUrl() { - let params = {}; - let export_url = this.apiUrl + '?format=csv'; + let exportUrl = new URL(this.apiUrl, document.location); + let exportUrlParams = exportUrl.searchParams; + exportUrlParams.set('format', 'csv'); if (!this.isLive) { - params.start = DateTime.fromJSDate(this.dateRange.startDate).toISO(); - params.end = DateTime.fromJSDate(this.dateRange.endDate).toISO(); - export_url += '&start=' + params.start + '&end=' + params.end; + exportUrlParams.set('start', DateTime.fromJSDate(this.dateRange.startDate).toISO()); + exportUrlParams.set('end', DateTime.fromJSDate(this.dateRange.endDate).toISO()); } - return export_url; + return exportUrl.toString(); }, totalListenerHours() { let tlh_seconds = 0; diff --git a/frontend/vue/components/Stations/Reports/Timeline.vue b/frontend/vue/components/Stations/Reports/Timeline.vue index b24ff35f2..e8f86ca7f 100644 --- a/frontend/vue/components/Stations/Reports/Timeline.vue +++ b/frontend/vue/components/Stations/Reports/Timeline.vue @@ -11,7 +11,7 @@ Download CSV - @@ -102,14 +102,21 @@ export default { }, computed: { apiUrl() { - let params = {}; - params.start = DateTime.fromJSDate(this.dateRange.startDate).toISODate(); - params.end = DateTime.fromJSDate(this.dateRange.endDate).toISODate(); + let apiUrl = new URL(this.baseApiUrl, document.location); - return this.baseApiUrl + '?start=' + params.start + '&end=' + params.end; + let apiUrlParams = apiUrl.searchParams; + apiUrlParams.set('start', DateTime.fromJSDate(this.dateRange.startDate).toISO()); + apiUrlParams.set('end', DateTime.fromJSDate(this.dateRange.endDate).toISO()); + + return apiUrl.toString(); }, exportUrl() { - return this.apiUrl + '&format=csv'; + let exportUrl = new URL(this.apiUrl, document.location); + let exportUrlParams = exportUrl.searchParams; + + exportUrlParams.set('format', 'csv'); + + return exportUrl.toString(); }, }, methods: { diff --git a/src/Controller/Api/Stations/HistoryController.php b/src/Controller/Api/Stations/HistoryController.php index 15cc57995..4eb7d2d6d 100644 --- a/src/Controller/Api/Stations/HistoryController.php +++ b/src/Controller/Api/Stations/HistoryController.php @@ -6,13 +6,16 @@ namespace App\Controller\Api\Stations; use App; use App\Entity; +use App\Environment; use App\Http\Response; use App\Http\ServerRequest; use App\OpenApi; -use App\Utilities\Csv; +use App\Utilities\File; use Azura\DoctrineBatchUtils\ReadOnlyBatchIteratorAggregate; use Carbon\CarbonImmutable; use Doctrine\ORM\EntityManagerInterface; +use Doctrine\ORM\Query; +use League\Csv\Writer; use OpenApi\Attributes as OA; use Psr\Http\Message\ResponseInterface; @@ -59,7 +62,8 @@ class HistoryController { public function __construct( protected EntityManagerInterface $em, - protected Entity\ApiGenerator\SongHistoryApiGenerator $songHistoryApiGenerator + protected Entity\ApiGenerator\SongHistoryApiGenerator $songHistoryApiGenerator, + protected Environment $environment ) { } @@ -69,13 +73,15 @@ class HistoryController */ public function __invoke(ServerRequest $request, Response $response): ResponseInterface { + set_time_limit($this->environment->getSyncLongExecutionTime()); + $station = $request->getStation(); $station_tz = $station->getTimezoneObject(); $params = $request->getQueryParams(); - if (!empty($params['start'])) { - $start = CarbonImmutable::parse($params['start'] . ' 00:00:00', $station_tz); - $end = CarbonImmutable::parse(($params['end'] ?? $params['start']) . ' 23:59:59', $station_tz); + if (!empty($params['start']) && !empty($params['end'])) { + $start = CarbonImmutable::parse($params['start'], $station_tz); + $end = CarbonImmutable::parse($params['end'], $station_tz); } else { $start = CarbonImmutable::parse('-2 weeks', $station_tz); $end = CarbonImmutable::now($station_tz); @@ -98,55 +104,19 @@ class HistoryController $format = $params['format'] ?? 'json'; if ('csv' === $format) { - $export_all = []; - $export_all[] = [ - 'Date', - 'Time', - 'Listeners', - 'Delta', - 'Track', - 'Artist', - 'Playlist', - 'Streamer', - ]; - - foreach (ReadOnlyBatchIteratorAggregate::fromQuery($qb->getQuery(), 100) as $sh) { - /** @var Entity\SongHistory $sh */ - $datetime = CarbonImmutable::createFromTimestamp($sh->getTimestampStart(), $station_tz); - - $playlist = $sh->getPlaylist(); - $playlistName = (null !== $playlist) - ? $playlist->getName() - : ''; - - $streamer = $sh->getStreamer(); - $streamerName = (null !== $streamer) - ? $streamer->getDisplayName() - : ''; - - $export_row = [ - $datetime->format('Y-m-d'), - $datetime->format('g:ia'), - $sh->getListenersStart(), - $sh->getDeltaTotal(), - $sh->getTitle() ?: $sh->getText(), - $sh->getArtist(), - $playlistName, - $streamerName, - ]; - - $export_all[] = $export_row; - } - - $csv_file = Csv::arrayToCsv($export_all); - $csv_filename = sprintf( + $csvFilename = sprintf( '%s_timeline_%s_to_%s.csv', $station->getShortName(), - $start->format('Ymd'), - $end->format('Ymd') + $start->format('Y-m-d_H-i-s'), + $end->format('Y-m-d_H-i-s') ); - return $response->renderStringAsFile($csv_file, 'text/csv', $csv_filename); + return $this->exportReportAsCsv( + $response, + $station, + $qb->getQuery(), + $csvFilename + ); } $search_phrase = trim($params['searchPhrase'] ?? ''); @@ -173,4 +143,59 @@ class HistoryController return $paginator->write($response); } + + protected function exportReportAsCsv( + Response $response, + Entity\Station $station, + Query $query, + string $filename + ): ResponseInterface { + $tempFile = File::generateTempPath($filename); + + $csv = Writer::createFromPath($tempFile, 'w+'); + + $csv->insertOne( + [ + 'Date', + 'Time', + 'Listeners', + 'Delta', + 'Track', + 'Artist', + 'Playlist', + 'Streamer', + ] + ); + + /** @var Entity\SongHistory $sh */ + foreach (ReadOnlyBatchIteratorAggregate::fromQuery($query, 100) as $sh) { + $datetime = CarbonImmutable::createFromTimestamp( + $sh->getTimestampStart(), + $station->getTimezoneObject() + ); + + $playlist = $sh->getPlaylist(); + $playlistName = (null !== $playlist) + ? $playlist->getName() + : ''; + + $streamer = $sh->getStreamer(); + $streamerName = (null !== $streamer) + ? $streamer->getDisplayName() + : ''; + + $csv->insertOne([ + $datetime->format('Y-m-d'), + $datetime->format('g:ia'), + $sh->getListenersStart(), + $sh->getDeltaTotal(), + $sh->getTitle() ?: $sh->getText(), + $sh->getArtist(), + $playlistName, + $streamerName, + ]); + } + + return $response->withFileDownload($tempFile, $filename, 'text/csv'); + } } diff --git a/src/Controller/Api/Stations/ListenersAction.php b/src/Controller/Api/Stations/ListenersAction.php index d6c367dda..82a5a1ab2 100644 --- a/src/Controller/Api/Stations/ListenersAction.php +++ b/src/Controller/Api/Stations/ListenersAction.php @@ -12,14 +12,13 @@ use App\Http\ServerRequest; use App\OpenApi; use App\Service\DeviceDetector; use App\Service\IpGeolocation; +use App\Utilities\File; use Azura\DoctrineBatchUtils\ReadOnlyBatchIteratorAggregate; use Carbon\CarbonImmutable; use Doctrine\ORM\EntityManagerInterface; -use GuzzleHttp\Psr7\Stream; use League\Csv\Writer; use OpenApi\Attributes as OA; use Psr\Http\Message\ResponseInterface; -use RuntimeException; #[ OA\Get( @@ -235,8 +234,9 @@ class ListenersAction array $listeners, string $filename ): ResponseInterface { - $tempFile = tmpfile() ?: throw new RuntimeException('Could not create temp file.'); - $csv = Writer::createFromStream($tempFile); + $tempFile = File::generateTempPath($filename); + + $csv = Writer::createFromPath($tempFile, 'w+'); $tz = $station->getTimezoneObject(); @@ -262,7 +262,7 @@ class ListenersAction $startTime = CarbonImmutable::createFromTimestamp($listener->connected_on, $tz); $endTime = CarbonImmutable::createFromTimestamp($listener->connected_until, $tz); - $export_row = [ + $exportRow = [ $listener->ip, $startTime->toIso8601String(), $endTime->toIso8601String(), @@ -273,31 +273,29 @@ class ListenersAction ]; if ('' === $listener->mount_name) { - $export_row[] = 'Unknown'; - $export_row[] = 'Unknown'; + $exportRow[] = 'Unknown'; + $exportRow[] = 'Unknown'; } else { - $export_row[] = ($listener->mount_is_local) ? 'Local' : 'Remote'; - $export_row[] = $listener->mount_name; + $exportRow[] = ($listener->mount_is_local) ? 'Local' : 'Remote'; + $exportRow[] = $listener->mount_name; } $location = $listener->location; if ('success' === $location['status']) { - $export_row[] = $location['region'] . ', ' . $location['country']; - $export_row[] = $location['country']; - $export_row[] = $location['region']; - $export_row[] = $location['city']; + $exportRow[] = $location['region'] . ', ' . $location['country']; + $exportRow[] = $location['country']; + $exportRow[] = $location['region']; + $exportRow[] = $location['city']; } else { - $export_row[] = $location['message'] ?? 'N/A'; - $export_row[] = ''; - $export_row[] = ''; - $export_row[] = ''; + $exportRow[] = $location['message'] ?? 'N/A'; + $exportRow[] = ''; + $exportRow[] = ''; + $exportRow[] = ''; } - $csv->insertOne($export_row); + $csv->insertOne($exportRow); } - $stream = new Stream($tempFile); - - return $response->renderStreamAsFile($stream, 'text/csv', $filename); + return $response->withFileDownload($tempFile, $filename, 'text/csv'); } } diff --git a/src/Controller/Api/Stations/Reports/PerformanceAction.php b/src/Controller/Api/Stations/Reports/PerformanceAction.php index 5c8e90c87..b7a07b626 100644 --- a/src/Controller/Api/Stations/Reports/PerformanceAction.php +++ b/src/Controller/Api/Stations/Reports/PerformanceAction.php @@ -8,8 +8,8 @@ use App\Http\Response; use App\Http\ServerRequest; use App\Paginator; use App\Sync\Task\RunAutomatedAssignmentTask; -use App\Utilities\Csv; -use Doctrine\ORM\EntityManagerInterface; +use App\Utilities\File; +use League\Csv\Writer; use Psr\Http\Message\ResponseInterface; class PerformanceAction @@ -17,20 +17,19 @@ class PerformanceAction public function __invoke( ServerRequest $request, Response $response, - EntityManagerInterface $em, RunAutomatedAssignmentTask $automationTask ): ResponseInterface { $station = $request->getStation(); - $automation_config = (array)$station->getAutomationSettings(); - $threshold_days = (int)($automation_config['threshold_days'] + $automationConfig = (array)$station->getAutomationSettings(); + $thresholdDays = (int)($automationConfig['threshold_days'] ?? RunAutomatedAssignmentTask::DEFAULT_THRESHOLD_DAYS); - $report_data = $automationTask->generateReport($station, $threshold_days); + $reportData = $automationTask->generateReport($station, $thresholdDays); // Do not show songs that are not in playlists. - $report_data = array_filter( - $report_data, + $reportData = array_filter( + $reportData, static function ($media) { return !(empty($media['playlists'])); } @@ -40,47 +39,63 @@ class PerformanceAction $format = $params['format'] ?? 'json'; if ($format === 'csv') { - $export_csv = [ - [ - 'Song Title', - 'Song Artist', - 'Filename', - 'Length', - 'Current Playlist', - 'Delta Joins', - 'Delta Losses', - 'Delta Total', - 'Play Count', - 'Play Percentage', - 'Weighted Ratio', - ], - ]; - - foreach ($report_data as $row) { - $export_csv[] = [ - $row['title'], - $row['artist'], - $row['path'], - $row['length'], - - implode('/', $row['playlists']), - $row['delta_positive'], - $row['delta_negative'], - $row['delta_total'], - - $row['num_plays'], - $row['percent_plays'] . '%', - $row['ratio'], - ]; - } - - $csv_file = Csv::arrayToCsv($export_csv); - $csv_filename = $station->getShortName() . '_media_' . date('Ymd') . '.csv'; - - return $response->renderStringAsFile($csv_file, 'text/csv', $csv_filename); + return $this->exportReportAsCsv( + $response, + $reportData, + $station->getShortName() . '_media_' . date('Ymd') . '.csv' + ); } - $paginator = Paginator::fromArray($report_data, $request); + $paginator = Paginator::fromArray($reportData, $request); return $paginator->write($response); } + + /** + * @param Response $response + * @param mixed[] $reportData + * @param string $filename + */ + protected function exportReportAsCsv( + Response $response, + array $reportData, + string $filename + ): ResponseInterface { + $tempFile = File::generateTempPath($filename); + + $csv = Writer::createFromPath($tempFile, 'w+'); + + $csv->insertOne( + [ + 'Song Title', + 'Song Artist', + 'Filename', + 'Length', + 'Current Playlist', + 'Delta Joins', + 'Delta Losses', + 'Delta Total', + 'Play Count', + 'Play Percentage', + 'Weighted Ratio', + ] + ); + + foreach ($reportData as $row) { + $csv->insertOne([ + $row['title'], + $row['artist'], + $row['path'], + $row['length'], + implode('/', $row['playlists']), + $row['delta_positive'], + $row['delta_negative'], + $row['delta_total'], + $row['num_plays'], + $row['percent_plays'] . '%', + $row['ratio'], + ]); + } + + return $response->withFileDownload($tempFile, $filename, 'text/csv'); + } } diff --git a/src/Utilities/Csv.php b/src/Utilities/Csv.php deleted file mode 100644 index ed249f432..000000000 --- a/src/Utilities/Csv.php +++ /dev/null @@ -1,43 +0,0 @@ - $table_val) { - if (!$headers_first_row && $row_count === 1) { - $header_row[] = '"' . str_replace('"', '""', $table_col) . '"'; - } - - $body_row[] = '"' . str_replace('"', '""', (string) $table_val) . '"'; - } - - if ($header_row) { - $final_display[] = implode(',', $header_row); - } - - if ($body_row) { - $final_display[] = implode(',', $body_row); - } - } - - return implode("\n", $final_display); - } -} diff --git a/src/Utilities/Xml.php b/src/Utilities/Xml.php deleted file mode 100644 index ed89001fa..000000000 --- a/src/Utilities/Xml.php +++ /dev/null @@ -1,99 +0,0 @@ -'); - self::arrToXml($array, $xml_info); - - return $xml_info->asXML(); - } - - /** - * @return mixed[] - */ - protected static function structToArray(mixed $values, mixed &$i): array - { - $child = []; - if (isset($values[$i]['value'])) { - $child[] = $values[$i]['value']; - } - - while ($i++ < count($values)) { - switch ($values[$i]['type']) { - case 'cdata': - $child[] = $values[$i]['value']; - break; - - case 'complete': - $name = $values[$i]['tag']; - if (!empty($name)) { - $child[$name] = $values[$i]['attributes'] ?? (($values[$i]['value']) ?: ''); - } - break; - - case 'open': - $name = $values[$i]['tag']; - $size = isset($child[$name]) ? sizeof($child[$name]) : 0; - $child[$name][$size] = self::structToArray($values, $i); - break; - - case 'close': - return $child; - } - } - - return $child; - } - - /** @noinspection PhpParameterByRefIsNotUsedAsReferenceInspection */ - protected static function arrToXml(array $array, SimpleXMLElement &$xml): void - { - foreach ($array as $key => $value) { - $key = is_numeric($key) ? "item$key" : $key; - if (is_array($value)) { - $subnode = $xml->addChild((string)$key); - - self::arrToXml($value, $subnode); - } else { - $xml->addChild((string)$key, htmlspecialchars($value, ENT_QUOTES | ENT_HTML5)); - } - } - } -} diff --git a/tests/Functional/Api_Stations_ReportsCest.php b/tests/Functional/Api_Stations_ReportsCest.php index e6d1a72c0..e520fa4d6 100644 --- a/tests/Functional/Api_Stations_ReportsCest.php +++ b/tests/Functional/Api_Stations_ReportsCest.php @@ -2,8 +2,12 @@ namespace Functional; +use Codeception\Util\Shared\Asserts; + class Api_Stations_ReportsCest extends CestAbstract { + use Asserts; + /** * @before setupComplete * @before login @@ -27,4 +31,136 @@ class Api_Stations_ReportsCest extends CestAbstract $I->seeResponseCodeIs(200); } + + /** + * @before setupComplete + * @before login + */ + public function downloadListenerReportsCsv(\FunctionalTester $I): void + { + $I->wantTo('Download station listener report CSV via API.'); + + $station = $this->getTestStation(); + $uriBase = '/api/station/' . $station->getId(); + + $startDateTime = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days')); + $endDateTime = new \DateTime(); + + $requestUrl = $uriBase . '/listeners?' . http_build_query( + [ + 'format' => 'csv', + 'start' => $startDateTime->format('Y-m-d\TH:i:s.v\Z'), + 'end' => $endDateTime->format('Y-m-d\TH:i:s.v\Z'), + ] + ); + + $csvHeaders = [ + 'IP', + 'Start Time', + 'End Time', + 'Seconds Connected', + 'User Agent', + 'Client', + 'Is Mobile', + 'Mount Type', + 'Mount Name', + 'Location', + 'Country', + 'Region', + 'City', + ]; + + $this->testReportCsv($I, $requestUrl, $csvHeaders); + } + + /** + * @before setupComplete + * @before login + */ + public function downloadHistoryReportCsv(\FunctionalTester $I): void + { + $I->wantTo('Download station timeline report CSV via API.'); + + $station = $this->getTestStation(); + $uriBase = '/api/station/' . $station->getId(); + + $startDateTime = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days')); + $endDateTime = new \DateTime(); + + $requestUrl = $uriBase . '/history?' . http_build_query( + [ + 'format' => 'csv', + 'start' => $startDateTime->format('Y-m-d\TH:i:s.v\Z'), + 'end' => $endDateTime->format('Y-m-d\TH:i:s.v\Z'), + ] + ); + + $csvHeaders = [ + 'Date', + 'Time', + 'Listeners', + 'Delta', + 'Track', + 'Artist', + 'Playlist', + 'Streamer', + ]; + + $this->testReportCsv($I, $requestUrl, $csvHeaders); + } + + /** + * @before setupComplete + * @before login + */ + public function downloadPerformanceReportCsv(\FunctionalTester $I): void + { + $I->wantTo('Download station song impact CSV via API.'); + + $station = $this->getTestStation(); + $uriBase = '/api/station/' . $station->getId(); + $requestUrl = $uriBase . '/reports/performance?format=csv'; + + $csvHeaders = [ + 'Song Title', + 'Song Artist', + 'Filename', + 'Length', + 'Current Playlist', + 'Delta Joins', + 'Delta Losses', + 'Delta Total', + 'Play Count', + 'Play Percentage', + 'Weighted Ratio', + ]; + + $this->testReportCsv($I, $requestUrl, $csvHeaders); + } + + protected function testReportCsv( + \FunctionalTester $I, + string $url, + array $headerFields + ): void { + $I->sendGet($url); + + $response = $I->grabResponse(); + + $responseCsv = str_getcsv($response); + + $this->assertIsArray($responseCsv); + $this->assertTrue( + count($responseCsv) > 0, + 'CSV is not empty' + ); + + foreach ($headerFields as $csvHeaderField) { + $this->assertContains( + $csvHeaderField, + $responseCsv, + 'CSV has header field' + ); + } + } } diff --git a/tests/Unit/ExportsTest.php b/tests/Unit/ExportsTest.php deleted file mode 100644 index 98ba05c47..000000000 --- a/tests/Unit/ExportsTest.php +++ /dev/null @@ -1,34 +0,0 @@ - 'Test Field A', - 'test_field_b' => 'Test Field B', - ], - ]; - - $csv = Utilities\Csv::arrayToCsv($raw_data, false); - $this->assertStringContainsString('"test_field_a","test_field_b"', $csv); - - $raw_data = 'Contents'; - $xml_array = Utilities\Xml::xmlToArray($raw_data); - - $this->assertArrayHasKey('test', $xml_array); - - $xml = Utilities\Xml::arrayToXml($xml_array); - - $this->assertStringContainsString($raw_data, $xml); - } -} diff --git a/tests/Unit/XmlTest.php b/tests/Unit/XmlTest.php index 39caa9b1a..1e1b193e5 100644 --- a/tests/Unit/XmlTest.php +++ b/tests/Unit/XmlTest.php @@ -4,10 +4,12 @@ namespace Unit; use App\Xml\Reader; use App\Xml\Writer; +use Codeception\Test\Unit; +use UnitTester; -class XmlTest extends \Codeception\Test\Unit +class XmlTest extends Unit { - protected \UnitTester $tester; + protected UnitTester $tester; public function testXml(): void {