tilderadio
/
AzuraCast
mirror of https://github.com/AzuraCast/AzuraCast.git
4
0
Fork 0
Code Issues Releases Wiki Activity

Update report exports to use league csv (#5132)

This commit is contained in:
Vaalyn 2022-03-22 05:35:35 +01:00 committed by GitHub
parent f35a4cb992
commit 04bd45fc2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 325 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
frontend/scss/vendors/_daterangepicker-colors.scss vendored
View File

@ -25,6 +25,10 @@
.calendar-table {
background: none;
border: 0;
.next span, .prev span {
border-color: $text-color;
}
}
td.off {
@ -39,4 +43,10 @@
th.available:hover {
background-color: $navdrawer-nav-link-bg-hover;
}
.monthselect, .yearselect, input, select.ampmselect, select.hourselect, select.minuteselect, select.secondselect {
background-color: $menu-bg;
border: 1px solid $menu-divider-bg;
color: $text-color;
}
}

12
frontend/vue/components/Stations/Reports/Listeners.vue
View File

@ -202,16 +202,16 @@ export default {
return DateTime.fromJSDate(this.liveTime).equals(DateTime.fromJSDate(this.dateRange.startDate));
},
exportUrl() {
let params = {};
let export_url = this.apiUrl + '?format=csv';
let exportUrl = new URL(this.apiUrl, document.location);
let exportUrlParams = exportUrl.searchParams;
exportUrlParams.set('format', 'csv');
if (!this.isLive) {
params.start = DateTime.fromJSDate(this.dateRange.startDate).toISO();
params.end = DateTime.fromJSDate(this.dateRange.endDate).toISO();
export_url += '&start=' + params.start + '&end=' + params.end;
exportUrlParams.set('start', DateTime.fromJSDate(this.dateRange.startDate).toISO());
exportUrlParams.set('end', DateTime.fromJSDate(this.dateRange.endDate).toISO());
}
return export_url;
return exportUrl.toString();
},
totalListenerHours() {
let tlh_seconds = 0;

19
frontend/vue/components/Stations/Reports/Timeline.vue
View File

@ -11,7 +11,7 @@
<translate key="lang_download_csv_button">Download CSV</translate>
</a>
<date-range-dropdown v-model="dateRange" :tz="stationTimeZone"
<date-range-dropdown time-picker v-model="dateRange" :tz="stationTimeZone"
@update="relist"></date-range-dropdown>
</div>
</div>
@ -102,14 +102,21 @@ export default {
},
computed: {
apiUrl() {
let params = {};
params.start = DateTime.fromJSDate(this.dateRange.startDate).toISODate();
params.end = DateTime.fromJSDate(this.dateRange.endDate).toISODate();
let apiUrl = new URL(this.baseApiUrl, document.location);
return this.baseApiUrl + '?start=' + params.start + '&end=' + params.end;
let apiUrlParams = apiUrl.searchParams;
apiUrlParams.set('start', DateTime.fromJSDate(this.dateRange.startDate).toISO());
apiUrlParams.set('end', DateTime.fromJSDate(this.dateRange.endDate).toISO());
return apiUrl.toString();
},
exportUrl() {
return this.apiUrl + '&format=csv';
let exportUrl = new URL(this.apiUrl, document.location);
let exportUrlParams = exportUrl.searchParams;
exportUrlParams.set('format', 'csv');
return exportUrl.toString();
},
},
methods: {

125
src/Controller/Api/Stations/HistoryController.php
View File

@ -6,13 +6,16 @@ namespace App\Controller\Api\Stations;
use App;
use App\Entity;
use App\Environment;
use App\Http\Response;
use App\Http\ServerRequest;
use App\OpenApi;
use App\Utilities\Csv;
use App\Utilities\File;
use Azura\DoctrineBatchUtils\ReadOnlyBatchIteratorAggregate;
use Carbon\CarbonImmutable;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\Query;
use League\Csv\Writer;
use OpenApi\Attributes as OA;
use Psr\Http\Message\ResponseInterface;
@ -59,7 +62,8 @@ class HistoryController
{
public function __construct(
protected EntityManagerInterface $em,
protected Entity\ApiGenerator\SongHistoryApiGenerator $songHistoryApiGenerator
protected Entity\ApiGenerator\SongHistoryApiGenerator $songHistoryApiGenerator,
protected Environment $environment
) {
}
@ -69,13 +73,15 @@ class HistoryController
*/
public function __invoke(ServerRequest $request, Response $response): ResponseInterface
{
set_time_limit($this->environment->getSyncLongExecutionTime());
$station = $request->getStation();
$station_tz = $station->getTimezoneObject();
$params = $request->getQueryParams();
if (!empty($params['start'])) {
$start = CarbonImmutable::parse($params['start'] . ' 00:00:00', $station_tz);
$end = CarbonImmutable::parse(($params['end'] ?? $params['start']) . ' 23:59:59', $station_tz);
if (!empty($params['start']) && !empty($params['end'])) {
$start = CarbonImmutable::parse($params['start'], $station_tz);
$end = CarbonImmutable::parse($params['end'], $station_tz);
} else {
$start = CarbonImmutable::parse('-2 weeks', $station_tz);
$end = CarbonImmutable::now($station_tz);
@ -98,55 +104,19 @@ class HistoryController
$format = $params['format'] ?? 'json';
if ('csv' === $format) {
$export_all = [];
$export_all[] = [
'Date',
'Time',
'Listeners',
'Delta',
'Track',
'Artist',
'Playlist',
'Streamer',
];
foreach (ReadOnlyBatchIteratorAggregate::fromQuery($qb->getQuery(), 100) as $sh) {
/** @var Entity\SongHistory $sh */
$datetime = CarbonImmutable::createFromTimestamp($sh->getTimestampStart(), $station_tz);
$playlist = $sh->getPlaylist();
$playlistName = (null !== $playlist)
? $playlist->getName()
: '';
$streamer = $sh->getStreamer();
$streamerName = (null !== $streamer)
? $streamer->getDisplayName()
: '';
$export_row = [
$datetime->format('Y-m-d'),
$datetime->format('g:ia'),
$sh->getListenersStart(),
$sh->getDeltaTotal(),
$sh->getTitle() ?: $sh->getText(),
$sh->getArtist(),
$playlistName,
$streamerName,
];
$export_all[] = $export_row;
}
$csv_file = Csv::arrayToCsv($export_all);
$csv_filename = sprintf(
$csvFilename = sprintf(
'%s_timeline_%s_to_%s.csv',
$station->getShortName(),
$start->format('Ymd'),
$end->format('Ymd')
$start->format('Y-m-d_H-i-s'),
$end->format('Y-m-d_H-i-s')
);
return $response->renderStringAsFile($csv_file, 'text/csv', $csv_filename);
return $this->exportReportAsCsv(
$response,
$station,
$qb->getQuery(),
$csvFilename
);
}
$search_phrase = trim($params['searchPhrase'] ?? '');
@ -173,4 +143,59 @@ class HistoryController
return $paginator->write($response);
}
protected function exportReportAsCsv(
Response $response,
Entity\Station $station,
Query $query,
string $filename
): ResponseInterface {
$tempFile = File::generateTempPath($filename);
$csv = Writer::createFromPath($tempFile, 'w+');
$csv->insertOne(
[
'Date',
'Time',
'Listeners',
'Delta',
'Track',
'Artist',
'Playlist',
'Streamer',
]
);
/** @var Entity\SongHistory $sh */
foreach (ReadOnlyBatchIteratorAggregate::fromQuery($query, 100) as $sh) {
$datetime = CarbonImmutable::createFromTimestamp(
$sh->getTimestampStart(),
$station->getTimezoneObject()
);
$playlist = $sh->getPlaylist();
$playlistName = (null !== $playlist)
? $playlist->getName()
: '';
$streamer = $sh->getStreamer();
$streamerName = (null !== $streamer)
? $streamer->getDisplayName()
: '';
$csv->insertOne([
$datetime->format('Y-m-d'),
$datetime->format('g:ia'),
$sh->getListenersStart(),
$sh->getDeltaTotal(),
$sh->getTitle() ?: $sh->getText(),
$sh->getArtist(),
$playlistName,
$streamerName,
]);
}
return $response->withFileDownload($tempFile, $filename, 'text/csv');
}
}

40
src/Controller/Api/Stations/ListenersAction.php
View File

@ -12,14 +12,13 @@ use App\Http\ServerRequest;
use App\OpenApi;
use App\Service\DeviceDetector;
use App\Service\IpGeolocation;
use App\Utilities\File;
use Azura\DoctrineBatchUtils\ReadOnlyBatchIteratorAggregate;
use Carbon\CarbonImmutable;
use Doctrine\ORM\EntityManagerInterface;
use GuzzleHttp\Psr7\Stream;
use League\Csv\Writer;
use OpenApi\Attributes as OA;
use Psr\Http\Message\ResponseInterface;
use RuntimeException;
#[
OA\Get(
@ -235,8 +234,9 @@ class ListenersAction
array $listeners,
string $filename
): ResponseInterface {
$tempFile = tmpfile() ?: throw new RuntimeException('Could not create temp file.');
$csv = Writer::createFromStream($tempFile);
$tempFile = File::generateTempPath($filename);
$csv = Writer::createFromPath($tempFile, 'w+');
$tz = $station->getTimezoneObject();
@ -262,7 +262,7 @@ class ListenersAction
$startTime = CarbonImmutable::createFromTimestamp($listener->connected_on, $tz);
$endTime = CarbonImmutable::createFromTimestamp($listener->connected_until, $tz);
$export_row = [
$exportRow = [
$listener->ip,
$startTime->toIso8601String(),
$endTime->toIso8601String(),
@ -273,31 +273,29 @@ class ListenersAction
];
if ('' === $listener->mount_name) {
$export_row[] = 'Unknown';
$export_row[] = 'Unknown';
$exportRow[] = 'Unknown';
$exportRow[] = 'Unknown';
} else {
$export_row[] = ($listener->mount_is_local) ? 'Local' : 'Remote';
$export_row[] = $listener->mount_name;
$exportRow[] = ($listener->mount_is_local) ? 'Local' : 'Remote';
$exportRow[] = $listener->mount_name;
}
$location = $listener->location;
if ('success' === $location['status']) {
$export_row[] = $location['region'] . ', ' . $location['country'];
$export_row[] = $location['country'];
$export_row[] = $location['region'];
$export_row[] = $location['city'];
$exportRow[] = $location['region'] . ', ' . $location['country'];
$exportRow[] = $location['country'];
$exportRow[] = $location['region'];
$exportRow[] = $location['city'];
} else {
$export_row[] = $location['message'] ?? 'N/A';
$export_row[] = '';
$export_row[] = '';
$export_row[] = '';
$exportRow[] = $location['message'] ?? 'N/A';
$exportRow[] = '';
$exportRow[] = '';
$exportRow[] = '';
}
$csv->insertOne($export_row);
$csv->insertOne($exportRow);
}
$stream = new Stream($tempFile);
return $response->renderStreamAsFile($stream, 'text/csv', $filename);
return $response->withFileDownload($tempFile, $filename, 'text/csv');
}
}

109
src/Controller/Api/Stations/Reports/PerformanceAction.php
View File

@ -8,8 +8,8 @@ use App\Http\Response;
use App\Http\ServerRequest;
use App\Paginator;
use App\Sync\Task\RunAutomatedAssignmentTask;
use App\Utilities\Csv;
use Doctrine\ORM\EntityManagerInterface;
use App\Utilities\File;
use League\Csv\Writer;
use Psr\Http\Message\ResponseInterface;
class PerformanceAction
@ -17,20 +17,19 @@ class PerformanceAction
public function __invoke(
ServerRequest $request,
Response $response,
EntityManagerInterface $em,
RunAutomatedAssignmentTask $automationTask
): ResponseInterface {
$station = $request->getStation();
$automation_config = (array)$station->getAutomationSettings();
$threshold_days = (int)($automation_config['threshold_days']
$automationConfig = (array)$station->getAutomationSettings();
$thresholdDays = (int)($automationConfig['threshold_days']
?? RunAutomatedAssignmentTask::DEFAULT_THRESHOLD_DAYS);
$report_data = $automationTask->generateReport($station, $threshold_days);
$reportData = $automationTask->generateReport($station, $thresholdDays);
// Do not show songs that are not in playlists.
$report_data = array_filter(
$report_data,
$reportData = array_filter(
$reportData,
static function ($media) {
return !(empty($media['playlists']));
}
@ -40,47 +39,63 @@ class PerformanceAction
$format = $params['format'] ?? 'json';
if ($format === 'csv') {
$export_csv = [
[
'Song Title',
'Song Artist',
'Filename',
'Length',
'Current Playlist',
'Delta Joins',
'Delta Losses',
'Delta Total',
'Play Count',
'Play Percentage',
'Weighted Ratio',
],
];
foreach ($report_data as $row) {
$export_csv[] = [
$row['title'],
$row['artist'],
$row['path'],
$row['length'],
implode('/', $row['playlists']),
$row['delta_positive'],
$row['delta_negative'],
$row['delta_total'],
$row['num_plays'],
$row['percent_plays'] . '%',
$row['ratio'],
];
}
$csv_file = Csv::arrayToCsv($export_csv);
$csv_filename = $station->getShortName() . '_media_' . date('Ymd') . '.csv';
return $response->renderStringAsFile($csv_file, 'text/csv', $csv_filename);
return $this->exportReportAsCsv(
$response,
$reportData,
$station->getShortName() . '_media_' . date('Ymd') . '.csv'
);
}
$paginator = Paginator::fromArray($report_data, $request);
$paginator = Paginator::fromArray($reportData, $request);
return $paginator->write($response);
}
/**
* @param Response $response
* @param mixed[] $reportData
* @param string $filename
*/
protected function exportReportAsCsv(
Response $response,
array $reportData,
string $filename
): ResponseInterface {
$tempFile = File::generateTempPath($filename);
$csv = Writer::createFromPath($tempFile, 'w+');
$csv->insertOne(
[
'Song Title',
'Song Artist',
'Filename',
'Length',
'Current Playlist',
'Delta Joins',
'Delta Losses',
'Delta Total',
'Play Count',
'Play Percentage',
'Weighted Ratio',
]
);
foreach ($reportData as $row) {
$csv->insertOne([
$row['title'],
$row['artist'],
$row['path'],
$row['length'],
implode('/', $row['playlists']),
$row['delta_positive'],
$row['delta_negative'],
$row['delta_total'],
$row['num_plays'],
$row['percent_plays'] . '%',
$row['ratio'],
]);
}
return $response->withFileDownload($tempFile, $filename, 'text/csv');
}
}

43
src/Utilities/Csv.php
View File

@ -1,43 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Utilities;
class Csv
{
/**
* Generate a CSV-compatible file body given an array.
*
* @param array $table_data
* @param bool $headers_first_row
*/
public static function arrayToCsv(array $table_data, bool $headers_first_row = true): string
{
$final_display = [];
$row_count = 0;
foreach ($table_data as $table_row) {
$row_count++;
$header_row = [];
$body_row = [];
foreach ($table_row as $table_col => $table_val) {
if (!$headers_first_row && $row_count === 1) {
$header_row[] = '"' . str_replace('"', '""', $table_col) . '"';
}
$body_row[] = '"' . str_replace('"', '""', (string) $table_val) . '"';
}
if ($header_row) {
$final_display[] = implode(',', $header_row);
}
if ($body_row) {
$final_display[] = implode(',', $body_row);
}
}
return implode("\n", $final_display);
}
}

99
src/Utilities/Xml.php
View File

@ -1,99 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Utilities;
use SimpleXMLElement;
class Xml
{
/**
* Convert from an XML string into a PHP array.
*
* @param string $xml
*
* @return mixed[]
*/
public static function xmlToArray(string $xml): array
{
$values = $index = $array = [];
$parser = xml_parser_create();
xml_parser_set_option($parser, XML_OPTION_SKIP_WHITE, 1);
xml_parser_set_option($parser, XML_OPTION_CASE_FOLDING, 0);
xml_parse_into_struct($parser, $xml, $values, $index);
xml_parser_free($parser);
$i = 0;
$name = $values[$i]['tag'];
$array[$name] = $values[$i]['attributes'] ?? '';
$array[$name] = self::structToArray($values, $i);
return $array;
}
/**
* Convert a PHP array into an XML string.
*
* @param array $array
*
*/
public static function arrayToXml(array $array): string|bool
{
$xml_info = new SimpleXMLElement('<?xml version="1.0"?><return></return>');
self::arrToXml($array, $xml_info);
return $xml_info->asXML();
}
/**
* @return mixed[]
*/
protected static function structToArray(mixed $values, mixed &$i): array
{
$child = [];
if (isset($values[$i]['value'])) {
$child[] = $values[$i]['value'];
}
while ($i++ < count($values)) {
switch ($values[$i]['type']) {
case 'cdata':
$child[] = $values[$i]['value'];
break;
case 'complete':
$name = $values[$i]['tag'];
if (!empty($name)) {
$child[$name] = $values[$i]['attributes'] ?? (($values[$i]['value']) ?: '');
}
break;
case 'open':
$name = $values[$i]['tag'];
$size = isset($child[$name]) ? sizeof($child[$name]) : 0;
$child[$name][$size] = self::structToArray($values, $i);
break;
case 'close':
return $child;
}
}
return $child;
}
/** @noinspection PhpParameterByRefIsNotUsedAsReferenceInspection */
protected static function arrToXml(array $array, SimpleXMLElement &$xml): void
{
foreach ($array as $key => $value) {
$key = is_numeric($key) ? "item$key" : $key;
if (is_array($value)) {
$subnode = $xml->addChild((string)$key);
self::arrToXml($value, $subnode);
} else {
$xml->addChild((string)$key, htmlspecialchars($value, ENT_QUOTES | ENT_HTML5));
}
}
}
}

136
tests/Functional/Api_Stations_ReportsCest.php
View File

@ -2,8 +2,12 @@
namespace Functional;
use Codeception\Util\Shared\Asserts;
class Api_Stations_ReportsCest extends CestAbstract
{
use Asserts;
/**
* @before setupComplete
* @before login
@ -27,4 +31,136 @@ class Api_Stations_ReportsCest extends CestAbstract
$I->seeResponseCodeIs(200);
}
/**
* @before setupComplete
* @before login
*/
public function downloadListenerReportsCsv(\FunctionalTester $I): void
{
$I->wantTo('Download station listener report CSV via API.');
$station = $this->getTestStation();
$uriBase = '/api/station/' . $station->getId();
$startDateTime = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'));
$endDateTime = new \DateTime();
$requestUrl = $uriBase . '/listeners?' . http_build_query(
[
'format' => 'csv',
'start' => $startDateTime->format('Y-m-d\TH:i:s.v\Z'),
'end' => $endDateTime->format('Y-m-d\TH:i:s.v\Z'),
]
);
$csvHeaders = [
'IP',
'Start Time',
'End Time',
'Seconds Connected',
'User Agent',
'Client',
'Is Mobile',
'Mount Type',
'Mount Name',
'Location',
'Country',
'Region',
'City',
];
$this->testReportCsv($I, $requestUrl, $csvHeaders);
}
/**
* @before setupComplete
* @before login
*/
public function downloadHistoryReportCsv(\FunctionalTester $I): void
{
$I->wantTo('Download station timeline report CSV via API.');
$station = $this->getTestStation();
$uriBase = '/api/station/' . $station->getId();
$startDateTime = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'));
$endDateTime = new \DateTime();
$requestUrl = $uriBase . '/history?' . http_build_query(
[
'format' => 'csv',
'start' => $startDateTime->format('Y-m-d\TH:i:s.v\Z'),
'end' => $endDateTime->format('Y-m-d\TH:i:s.v\Z'),
]
);
$csvHeaders = [
'Date',
'Time',
'Listeners',
'Delta',
'Track',
'Artist',
'Playlist',
'Streamer',
];
$this->testReportCsv($I, $requestUrl, $csvHeaders);
}
/**
* @before setupComplete
* @before login
*/
public function downloadPerformanceReportCsv(\FunctionalTester $I): void
{
$I->wantTo('Download station song impact CSV via API.');
$station = $this->getTestStation();
$uriBase = '/api/station/' . $station->getId();
$requestUrl = $uriBase . '/reports/performance?format=csv';
$csvHeaders = [
'Song Title',
'Song Artist',
'Filename',
'Length',
'Current Playlist',
'Delta Joins',
'Delta Losses',
'Delta Total',
'Play Count',
'Play Percentage',
'Weighted Ratio',
];
$this->testReportCsv($I, $requestUrl, $csvHeaders);
}
protected function testReportCsv(
\FunctionalTester $I,
string $url,
array $headerFields
): void {
$I->sendGet($url);
$response = $I->grabResponse();
$responseCsv = str_getcsv($response);
$this->assertIsArray($responseCsv);
$this->assertTrue(
count($responseCsv) > 0,
'CSV is not empty'
);
foreach ($headerFields as $csvHeaderField) {
$this->assertContains(
$csvHeaderField,
$responseCsv,
'CSV has header field'
);
}
}
}

34
tests/Unit/ExportsTest.php
View File

@ -1,34 +0,0 @@
<?php
namespace Unit;
use App\Utilities;
use Codeception\Test\Unit;
use UnitTester;
class ExportsTest extends Unit
{
protected UnitTester $tester;
public function testExports()
{
$raw_data = [
[
'test_field_a' => 'Test Field A',
'test_field_b' => 'Test Field B',
],
];
$csv = Utilities\Csv::arrayToCsv($raw_data, false);
$this->assertStringContainsString('"test_field_a","test_field_b"', $csv);
$raw_data = '<test><subtest>Contents</subtest></test>';
$xml_array = Utilities\Xml::xmlToArray($raw_data);
$this->assertArrayHasKey('test', $xml_array);
$xml = Utilities\Xml::arrayToXml($xml_array);
$this->assertStringContainsString($raw_data, $xml);
}
}

6
tests/Unit/XmlTest.php
View File

@ -4,10 +4,12 @@ namespace Unit;
use App\Xml\Reader;
use App\Xml\Writer;
use Codeception\Test\Unit;
use UnitTester;
class XmlTest extends \Codeception\Test\Unit
class XmlTest extends Unit
{
protected \UnitTester $tester;
protected UnitTester $tester;
public function testXml(): void
{