From 03fa0d1ff6dfd386fc1143902902b6b39a7e446d Mon Sep 17 00:00:00 2001 From: Buster Neece Date: Wed, 25 Jan 2023 05:58:52 -0600 Subject: [PATCH 01/11] Infrastructure setup for Meilisearch. --- Dockerfile | 5 +- composer.json | 1 + composer.lock | 458 +++++++++++++++++- docker-compose.sample.yml | 2 + util/docker/web/meilisearch/config.toml | 3 + util/docker/web/service.full/meilisearch.conf | 17 + util/docker/web/setup/00_packages.sh | 5 +- util/docker/web/setup/meilisearch.sh | 7 + 8 files changed, 494 insertions(+), 4 deletions(-) create mode 100644 util/docker/web/meilisearch/config.toml create mode 100644 util/docker/web/service.full/meilisearch.conf create mode 100644 util/docker/web/setup/meilisearch.sh diff --git a/Dockerfile b/Dockerfile index 3c11c03bd..ab1cf4a01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,7 @@ RUN bash /bd_build/redis/setup.sh \ RUN rm -rf /bd_build -VOLUME ["/var/azuracast/stations", "/var/azuracast/uploads", "/var/azuracast/backups", "/var/azuracast/sftpgo/persist", "/var/azuracast/servers/shoutcast2"] +VOLUME ["/var/azuracast/stations", "/var/azuracast/uploads", "/var/azuracast/backups", "/var/azuracast/sftpgo/persist", "/var/azuracast/servers/shoutcast2", "/var/azuracast/meilisearch/persist"] # # Final build (Just environment vars and squishing the FS) @@ -116,7 +116,8 @@ ENV TZ="UTC" \ PROFILING_EXTENSION_ALWAYS_ON=0 \ PROFILING_EXTENSION_HTTP_KEY=dev \ PROFILING_EXTENSION_HTTP_IP_WHITELIST=* \ - ENABLE_WEB_UPDATER="true" + ENABLE_WEB_UPDATER="true" \ + MEILI_MASTER_KEY="azur4c457" # Entrypoint and default command ENTRYPOINT ["tini", "--", "/usr/local/bin/my_init"] diff --git a/composer.json b/composer.json index 75490346f..755cdc0c7 100644 --- a/composer.json +++ b/composer.json @@ -52,6 +52,7 @@ "lstrojny/fxmlrpc": "dev-master", "marcw/rss-writer": "^0.4.0", "matomo/device-detector": "^6", + "meilisearch/meilisearch-php": "^0.27.0", "mezzio/mezzio-session": "^1.3", "mezzio/mezzio-session-cache": "^1.7", "monolog/monolog": "^3", diff --git a/composer.lock b/composer.lock index cc9b467ce..b918f8986 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b1487b626a93972c92e12cb2130e0454", + "content-hash": "f92319538865587280bea86d366629b8", "packages": [ { "name": "aws/aws-crt-php", @@ -434,6 +434,72 @@ ], "time": "2022-08-10T22:54:19+00:00" }, + { + "name": "clue/stream-filter", + "version": "v1.6.0", + "source": { + "type": "git", + "url": "https://github.com/clue/stream-filter.git", + "reference": "d6169430c7731d8509da7aecd0af756a5747b78e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/clue/stream-filter/zipball/d6169430c7731d8509da7aecd0af756a5747b78e", + "reference": "d6169430c7731d8509da7aecd0af756a5747b78e", + "shasum": "" + }, + "require": { + "php": ">=5.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3 || ^5.7 || ^4.8.36" + }, + "type": "library", + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "Clue\\StreamFilter\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Christian Lück", + "email": "christian@clue.engineering" + } + ], + "description": "A simple and modern approach to stream filtering in PHP", + "homepage": "https://github.com/clue/php-stream-filter", + "keywords": [ + "bucket brigade", + "callback", + "filter", + "php_user_filter", + "stream", + "stream_filter_append", + "stream_filter_register" + ], + "support": { + "issues": "https://github.com/clue/stream-filter/issues", + "source": "https://github.com/clue/stream-filter/tree/v1.6.0" + }, + "funding": [ + { + "url": "https://clue.engineering/support", + "type": "custom" + }, + { + "url": "https://github.com/clue", + "type": "github" + } + ], + "time": "2022-02-21T13:15:14+00:00" + }, { "name": "composer/ca-bundle", "version": "1.3.5", @@ -3742,6 +3808,74 @@ }, "time": "2023-01-11T09:41:57+00:00" }, + { + "name": "meilisearch/meilisearch-php", + "version": "v0.27.0", + "source": { + "type": "git", + "url": "https://github.com/meilisearch/meilisearch-php.git", + "reference": "e95db9ed85a45dcd831573979bf1d42cb99b9b3b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/meilisearch/meilisearch-php/zipball/e95db9ed85a45dcd831573979bf1d42cb99b9b3b", + "reference": "e95db9ed85a45dcd831573979bf1d42cb99b9b3b", + "shasum": "" + }, + "require": { + "ext-json": "*", + "php": "^7.4 || ^8.0", + "php-http/client-common": "^2.0", + "php-http/discovery": "^1.7", + "php-http/httplug": "^2.1" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.0", + "guzzlehttp/guzzle": "^7.1", + "http-interop/http-factory-guzzle": "^1.0", + "phpstan/extension-installer": "^1.1", + "phpstan/phpstan": "1.9.3", + "phpstan/phpstan-deprecation-rules": "^1.0", + "phpstan/phpstan-phpunit": "^1.0", + "phpstan/phpstan-strict-rules": "^1.1", + "phpunit/phpunit": "^9.5" + }, + "suggest": { + "guzzlehttp/guzzle": "Use Guzzle ^7 as HTTP client", + "http-interop/http-factory-guzzle": "Factory for guzzlehttp/guzzle" + }, + "type": "library", + "autoload": { + "psr-4": { + "MeiliSearch\\": "src/", + "Meilisearch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Clementine Urquizar", + "email": "clementine@meilisearch.com" + } + ], + "description": "PHP wrapper for the Meilisearch API", + "keywords": [ + "api", + "client", + "instant", + "meilisearch", + "php", + "search" + ], + "support": { + "issues": "https://github.com/meilisearch/meilisearch-php/issues", + "source": "https://github.com/meilisearch/meilisearch-php/tree/v0.27.0" + }, + "time": "2023-01-10T19:29:02+00:00" + }, { "name": "mezzio/mezzio-session", "version": "1.12.0", @@ -4907,6 +5041,81 @@ }, "time": "2022-12-09T13:57:05+00:00" }, + { + "name": "php-http/client-common", + "version": "2.6.0", + "source": { + "type": "git", + "url": "https://github.com/php-http/client-common.git", + "reference": "45db684cd4e186dcdc2b9c06b22970fe123796c0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/client-common/zipball/45db684cd4e186dcdc2b9c06b22970fe123796c0", + "reference": "45db684cd4e186dcdc2b9c06b22970fe123796c0", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0", + "php-http/httplug": "^2.0", + "php-http/message": "^1.6", + "php-http/message-factory": "^1.0", + "psr/http-client": "^1.0", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.0", + "symfony/options-resolver": "~4.0.15 || ~4.1.9 || ^4.2.1 || ^5.0 || ^6.0", + "symfony/polyfill-php80": "^1.17" + }, + "require-dev": { + "doctrine/instantiator": "^1.1", + "guzzlehttp/psr7": "^1.4", + "nyholm/psr7": "^1.2", + "phpspec/phpspec": "^5.1 || ^6.3 || ^7.1", + "phpspec/prophecy": "^1.10.2", + "phpunit/phpunit": "^7.5.15 || ^8.5 || ^9.3" + }, + "suggest": { + "ext-json": "To detect JSON responses with the ContentTypePlugin", + "ext-libxml": "To detect XML responses with the ContentTypePlugin", + "php-http/cache-plugin": "PSR-6 Cache plugin", + "php-http/logger-plugin": "PSR-3 Logger plugin", + "php-http/stopwatch-plugin": "Symfony Stopwatch plugin" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Http\\Client\\Common\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com" + } + ], + "description": "Common HTTP Client implementations and tools for HTTPlug", + "homepage": "http://httplug.io", + "keywords": [ + "client", + "common", + "http", + "httplug" + ], + "support": { + "issues": "https://github.com/php-http/client-common/issues", + "source": "https://github.com/php-http/client-common/tree/2.6.0" + }, + "time": "2022-09-29T09:59:43+00:00" + }, { "name": "php-http/discovery", "version": "1.14.3", @@ -4974,6 +5183,253 @@ }, "time": "2022-07-11T14:04:40+00:00" }, + { + "name": "php-http/httplug", + "version": "2.3.0", + "source": { + "type": "git", + "url": "https://github.com/php-http/httplug.git", + "reference": "f640739f80dfa1152533976e3c112477f69274eb" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/httplug/zipball/f640739f80dfa1152533976e3c112477f69274eb", + "reference": "f640739f80dfa1152533976e3c112477f69274eb", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0", + "php-http/promise": "^1.1", + "psr/http-client": "^1.0", + "psr/http-message": "^1.0" + }, + "require-dev": { + "friends-of-phpspec/phpspec-code-coverage": "^4.1", + "phpspec/phpspec": "^5.1 || ^6.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-4": { + "Http\\Client\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eric GELOEN", + "email": "geloen.eric@gmail.com" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://sagikazarmark.hu" + } + ], + "description": "HTTPlug, the HTTP client abstraction for PHP", + "homepage": "http://httplug.io", + "keywords": [ + "client", + "http" + ], + "support": { + "issues": "https://github.com/php-http/httplug/issues", + "source": "https://github.com/php-http/httplug/tree/2.3.0" + }, + "time": "2022-02-21T09:52:22+00:00" + }, + { + "name": "php-http/message", + "version": "1.13.0", + "source": { + "type": "git", + "url": "https://github.com/php-http/message.git", + "reference": "7886e647a30a966a1a8d1dad1845b71ca8678361" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/message/zipball/7886e647a30a966a1a8d1dad1845b71ca8678361", + "reference": "7886e647a30a966a1a8d1dad1845b71ca8678361", + "shasum": "" + }, + "require": { + "clue/stream-filter": "^1.5", + "php": "^7.1 || ^8.0", + "php-http/message-factory": "^1.0.2", + "psr/http-message": "^1.0" + }, + "provide": { + "php-http/message-factory-implementation": "1.0" + }, + "require-dev": { + "ergebnis/composer-normalize": "^2.6", + "ext-zlib": "*", + "guzzlehttp/psr7": "^1.0", + "laminas/laminas-diactoros": "^2.0", + "phpspec/phpspec": "^5.1 || ^6.3 || ^7.1", + "slim/slim": "^3.0" + }, + "suggest": { + "ext-zlib": "Used with compressor/decompressor streams", + "guzzlehttp/psr7": "Used with Guzzle PSR-7 Factories", + "laminas/laminas-diactoros": "Used with Diactoros Factories", + "slim/slim": "Used with Slim Framework PSR-7 implementation" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.10-dev" + } + }, + "autoload": { + "files": [ + "src/filters.php" + ], + "psr-4": { + "Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com" + } + ], + "description": "HTTP Message related tools", + "homepage": "http://php-http.org", + "keywords": [ + "http", + "message", + "psr-7" + ], + "support": { + "issues": "https://github.com/php-http/message/issues", + "source": "https://github.com/php-http/message/tree/1.13.0" + }, + "time": "2022-02-11T13:41:14+00:00" + }, + { + "name": "php-http/message-factory", + "version": "v1.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-http/message-factory.git", + "reference": "a478cb11f66a6ac48d8954216cfed9aa06a501a1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/message-factory/zipball/a478cb11f66a6ac48d8954216cfed9aa06a501a1", + "reference": "a478cb11f66a6ac48d8954216cfed9aa06a501a1", + "shasum": "" + }, + "require": { + "php": ">=5.4", + "psr/http-message": "^1.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "psr-4": { + "Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com" + } + ], + "description": "Factory interfaces for PSR-7 HTTP Message", + "homepage": "http://php-http.org", + "keywords": [ + "factory", + "http", + "message", + "stream", + "uri" + ], + "support": { + "issues": "https://github.com/php-http/message-factory/issues", + "source": "https://github.com/php-http/message-factory/tree/master" + }, + "time": "2015-12-19T14:08:53+00:00" + }, + { + "name": "php-http/promise", + "version": "1.1.0", + "source": { + "type": "git", + "url": "https://github.com/php-http/promise.git", + "reference": "4c4c1f9b7289a2ec57cde7f1e9762a5789506f88" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/promise/zipball/4c4c1f9b7289a2ec57cde7f1e9762a5789506f88", + "reference": "4c4c1f9b7289a2ec57cde7f1e9762a5789506f88", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0" + }, + "require-dev": { + "friends-of-phpspec/phpspec-code-coverage": "^4.3.2", + "phpspec/phpspec": "^5.1.2 || ^6.2" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-4": { + "Http\\Promise\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Joel Wurtz", + "email": "joel.wurtz@gmail.com" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com" + } + ], + "description": "Promise used for asynchronous HTTP requests", + "homepage": "http://httplug.io", + "keywords": [ + "promise" + ], + "support": { + "issues": "https://github.com/php-http/promise/issues", + "source": "https://github.com/php-http/promise/tree/1.1.0" + }, + "time": "2020-07-07T09:29:14+00:00" + }, { "name": "phpmyadmin/motranslator", "version": "5.3.0", diff --git a/docker-compose.sample.yml b/docker-compose.sample.yml index c5e10c068..16d0dc4dc 100644 --- a/docker-compose.sample.yml +++ b/docker-compose.sample.yml @@ -191,6 +191,7 @@ services: - stereo_tool_install:/var/azuracast/servers/stereo_tool - geolite_install:/var/azuracast/geoip - sftpgo_data:/var/azuracast/sftpgo/persist + - meilisearch_data:/var/azuracast/meilisearch/persist - backups:/var/azuracast/backups - acme:/var/azuracast/acme - db_data:/var/lib/mysql @@ -221,6 +222,7 @@ volumes: stereo_tool_install: { } geolite_install: { } sftpgo_data: { } + meilisearch_data: { } station_data: { } www_uploads: { } backups: { } diff --git a/util/docker/web/meilisearch/config.toml b/util/docker/web/meilisearch/config.toml new file mode 100644 index 000000000..94acead3f --- /dev/null +++ b/util/docker/web/meilisearch/config.toml @@ -0,0 +1,3 @@ +db_path = "/var/azuracast/meilisearch/persist" +http_addr = "localhost:6070" +env = "production" diff --git a/util/docker/web/service.full/meilisearch.conf b/util/docker/web/service.full/meilisearch.conf new file mode 100644 index 000000000..cf719b537 --- /dev/null +++ b/util/docker/web/service.full/meilisearch.conf @@ -0,0 +1,17 @@ +[program:meilisearch] +command=meilisearch --config-file-path=/var/azuracast/meilisearch/config.toml +priority=500 +numprocs=1 +autostart=true +autorestart=true + +stopasgroup=true +killasgroup=true + +stdout_logfile=/var/azuracast/www_tmp/service_meilisearch.log +stdout_logfile_maxbytes=5MB +stdout_logfile_backups=5 +redirect_stderr=true + +stdout_events_enabled = true +stderr_events_enabled = true diff --git a/util/docker/web/setup/00_packages.sh b/util/docker/web/setup/00_packages.sh index c74b6cce6..a26b60782 100644 --- a/util/docker/web/setup/00_packages.sh +++ b/util/docker/web/setup/00_packages.sh @@ -8,6 +8,8 @@ add-apt-repository -y ppa:chris-needham/ppa add-apt-repository -y ppa:sftpgo/sftpgo add-apt-repository -y ppa:ondrej/php +echo "deb [trusted=yes] https://apt.fury.io/meilisearch/ /" | sudo tee /etc/apt/sources.list.d/fury.list + apt-get update apt-get install -y --no-install-recommends \ @@ -16,4 +18,5 @@ apt-get install -y --no-install-recommends \ sftpgo \ tmpreaper \ zstd \ - netbase + netbase \ + meilisearch-http diff --git a/util/docker/web/setup/meilisearch.sh b/util/docker/web/setup/meilisearch.sh new file mode 100644 index 000000000..53e6ff9eb --- /dev/null +++ b/util/docker/web/setup/meilisearch.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e +set -x + +mkdir -p /var/azuracast/meilisearch/persist + +cp /bd_build/web/meilisearch/config.toml /var/azuracast/meilisearch/config.toml From 40e8589755541098698e90f3b7c6f4d1663831ea Mon Sep 17 00:00:00 2001 From: Buster Neece Date: Wed, 25 Jan 2023 19:26:44 -0600 Subject: [PATCH 02/11] PHP stuff. --- src/Entity/StationMedia.php | 9 ++ src/Environment.php | 9 ++ src/Message/AddMediaToSearchIndexMessage.php | 33 +++++++ src/Service/Meilisearch.php | 95 ++++++++++++++++++++ src/Service/ServiceControl.php | 1 + 5 files changed, 147 insertions(+) create mode 100644 src/Message/AddMediaToSearchIndexMessage.php create mode 100644 src/Service/Meilisearch.php diff --git a/src/Entity/StationMedia.php b/src/Entity/StationMedia.php index f4476ed96..738dd8c7d 100644 --- a/src/Entity/StationMedia.php +++ b/src/Entity/StationMedia.php @@ -192,6 +192,15 @@ class StationMedia implements ] protected int $art_updated_at = 0; + #[ + OA\Property( + description: "The latest time (UNIX timestamp) when the search record for this entry was updated.", + example: OpenApi::SAMPLE_TIMESTAMP + ), + ORM\Column + ] + protected int $search_updated_at = 0; + /** @var Collection */ #[ OA\Property(type: "array", items: new OA\Items()), diff --git a/src/Environment.php b/src/Environment.php index d580ddbbe..5edfa12f8 100644 --- a/src/Environment.php +++ b/src/Environment.php @@ -54,6 +54,8 @@ final class Environment public const ENABLE_WEB_UPDATER = 'ENABLE_WEB_UPDATER'; + public const MEILI_MASTER_KEY = 'MEILI_MASTER_KEY'; + // Database and Cache Configuration Variables public const DB_HOST = 'MYSQL_HOST'; public const DB_PORT = 'MYSQL_PORT'; @@ -90,6 +92,8 @@ final class Environment self::PROFILING_EXTENSION_HTTP_KEY => 'dev', self::ENABLE_WEB_UPDATER => false, + + self::MEILI_MASTER_KEY => 'azur4c457', ]; public function __construct(array $elements = []) @@ -370,6 +374,11 @@ final class Environment return $this->isDocker() && self::envToBool($this->data[self::ENABLE_WEB_UPDATER] ?? false); } + public function getMeiliMasterKey(): string + { + return $this->data[self::MEILI_MASTER_KEY] ?? $this->defaults[self::MEILI_MASTER_KEY]; + } + public static function getDefaultsForEnvironment(Environment $existingEnv): self { return new self([ diff --git a/src/Message/AddMediaToSearchIndexMessage.php b/src/Message/AddMediaToSearchIndexMessage.php new file mode 100644 index 000000000..989295033 --- /dev/null +++ b/src/Message/AddMediaToSearchIndexMessage.php @@ -0,0 +1,33 @@ + $this->storage_location_id, + 'media' => $this->media, + ], JSON_THROW_ON_ERROR) + ); + + return 'AddMediaToSearchIndexMessage_' . $messageHash; + } + + public function getQueue(): string + { + return QueueManagerInterface::QUEUE_MEDIA; + } +} diff --git a/src/Service/Meilisearch.php b/src/Service/Meilisearch.php new file mode 100644 index 000000000..39d8d9973 --- /dev/null +++ b/src/Service/Meilisearch.php @@ -0,0 +1,95 @@ +environment->isDocker(); + } + + public function getClient(): Client + { + static $client; + + if (!$this->isSupported()) { + throw new \RuntimeException('This feature is not supported on this installation.'); + } + + if (!isset($client)) { + $psrFactory = new HttpFactory(); + $client = new Client( + 'http://localhost:6070', + $this->environment->getMeiliMasterKey(), + $this->httpClient, + requestFactory: $psrFactory, + streamFactory: $psrFactory + ); + } + + return $client; + } + + public function setupIndex(StorageLocation $storageLocation): void + { + $indexSettings = [ + 'primaryKey' => 'id', + 'filterableAttributes' => [ + 'playlists', + 'is_requestable', + 'is_on_demand' + ], + 'sortableAttributes' => [ + 'path', + 'mtime', + 'length', + 'title', + 'artist', + 'album', + 'genre', + 'isrc', + ], + ]; + + foreach($this->customFieldRepo->getFieldIds() as $fieldId => $fieldShortCode) { + $indexSettings['sortableAttributes'][] = 'custom_field_'.$fieldId; + } + + $client = $this->getClient(); + $client->updateIndex( + self::getIndexId($storageLocation), + $indexSettings + ); + } + + public function addToIndex( + StorageLocation $storageLocation, + array $ids + ): void { + + } + + public static function getIndexId(StorageLocation $storageLocation): string + { + return 'media_'.$storageLocation->getIdRequired(); + } +} diff --git a/src/Service/ServiceControl.php b/src/Service/ServiceControl.php index f4287585e..2fdaf9e6e 100644 --- a/src/Service/ServiceControl.php +++ b/src/Service/ServiceControl.php @@ -84,6 +84,7 @@ final class ServiceControl 'redis' => __('Cache'), 'sftpgo' => __('SFTP service'), 'centrifugo' => __('Live Now Playing updates'), + 'meilisearch' => __('Meilisearch'), ]; if (!$this->centrifugo->isSupported()) { From 0bf0bcdb0280d5f4002f56ba2dcf058bedc86fff Mon Sep 17 00:00:00 2001 From: Buster Neece Date: Fri, 27 Jan 2023 03:30:32 -0600 Subject: [PATCH 03/11] Make Index a separate class. --- src/Service/Meilisearch.php | 52 ++++++------------------ src/Service/Meilisearch/Index.php | 66 +++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 src/Service/Meilisearch/Index.php diff --git a/src/Service/Meilisearch.php b/src/Service/Meilisearch.php index 39d8d9973..b73d1904f 100644 --- a/src/Service/Meilisearch.php +++ b/src/Service/Meilisearch.php @@ -4,9 +4,10 @@ declare(strict_types=1); namespace App\Service; -use App\Entity\Repository\CustomFieldRepository; use App\Entity\StorageLocation; use App\Environment; +use App\Service\Meilisearch\Index; +use DI\FactoryInterface; use GuzzleHttp\Client as GuzzleClient; use GuzzleHttp\Psr7\HttpFactory; use Meilisearch\Client; @@ -18,7 +19,7 @@ final class Meilisearch public function __construct( private readonly Environment $environment, private readonly GuzzleClient $httpClient, - private readonly CustomFieldRepository $customFieldRepo, + private readonly FactoryInterface $factory ) { } @@ -49,47 +50,20 @@ final class Meilisearch return $client; } - public function setupIndex(StorageLocation $storageLocation): void + public function getIndex(StorageLocation $storageLocation): Index { - $indexSettings = [ - 'primaryKey' => 'id', - 'filterableAttributes' => [ - 'playlists', - 'is_requestable', - 'is_on_demand' - ], - 'sortableAttributes' => [ - 'path', - 'mtime', - 'length', - 'title', - 'artist', - 'album', - 'genre', - 'isrc', - ], - ]; - - foreach($this->customFieldRepo->getFieldIds() as $fieldId => $fieldShortCode) { - $indexSettings['sortableAttributes'][] = 'custom_field_'.$fieldId; - } - - $client = $this->getClient(); - $client->updateIndex( - self::getIndexId($storageLocation), - $indexSettings + return $this->factory->make( + Index::class, + [ + 'storageLocation' => $storageLocation, + 'indexUid' => self::getIndexUid($storageLocation), + 'client' => $this->getClient(), + ] ); } - public function addToIndex( - StorageLocation $storageLocation, - array $ids - ): void { - - } - - public static function getIndexId(StorageLocation $storageLocation): string + public static function getIndexUid(StorageLocation $storageLocation): string { - return 'media_'.$storageLocation->getIdRequired(); + return 'media_' . $storageLocation->getIdRequired(); } } diff --git a/src/Service/Meilisearch/Index.php b/src/Service/Meilisearch/Index.php new file mode 100644 index 000000000..56d7f0249 --- /dev/null +++ b/src/Service/Meilisearch/Index.php @@ -0,0 +1,66 @@ + 'id', + 'filterableAttributes' => [ + 'playlists', + 'is_requestable', + 'is_on_demand', + ], + 'sortableAttributes' => [ + 'path', + 'mtime', + 'length', + 'title', + 'artist', + 'album', + 'genre', + 'isrc', + ], + ]; + + foreach ($this->customFieldRepo->getFieldIds() as $fieldId => $fieldShortCode) { + $indexSettings['sortableAttributes'][] = 'custom_field_' . $fieldId; + } + + $this->client->updateIndex( + $this->indexUid, + $indexSettings + ); + } + + /** @return Station[] */ + private function iterateStations(): iterable + { + return $this->em->createQuery( + <<<'DQL' + SELECT s FROM App\Entity\Station s + WHERE s.media_storage_location = :storageLocation + DQL + )->setParameter('storageLocation', $this->storageLocation) + ->toIterable(); + } +} From 5bd5c7741615a92b70970fa8cfbbe0beef510d3e Mon Sep 17 00:00:00 2001 From: Buster Neece Date: Fri, 27 Jan 2023 07:57:13 -0600 Subject: [PATCH 04/11] Initial indexing work for Meilisearch. --- src/Service/Meilisearch/Index.php | 159 +++++++++++++++++++++++++----- 1 file changed, 137 insertions(+), 22 deletions(-) diff --git a/src/Service/Meilisearch/Index.php b/src/Service/Meilisearch/Index.php index 56d7f0249..2a18dddfd 100644 --- a/src/Service/Meilisearch/Index.php +++ b/src/Service/Meilisearch/Index.php @@ -8,6 +8,7 @@ use App\Doctrine\ReloadableEntityManagerInterface; use App\Entity\Repository\CustomFieldRepository; use App\Entity\Station; use App\Entity\StorageLocation; +use App\Environment; use Meilisearch\Client; final class Index @@ -15,6 +16,7 @@ final class Index public function __construct( private readonly ReloadableEntityManagerInterface $em, private readonly CustomFieldRepository $customFieldRepo, + private readonly Environment $environment, private readonly Client $client, private readonly StorageLocation $storageLocation, private readonly string $indexUid @@ -23,32 +25,144 @@ final class Index public function configure(): void { - $indexSettings = [ - 'primaryKey' => 'id', - 'filterableAttributes' => [ - 'playlists', - 'is_requestable', - 'is_on_demand', - ], - 'sortableAttributes' => [ - 'path', - 'mtime', - 'length', - 'title', - 'artist', - 'album', - 'genre', - 'isrc', - ], + $filterableAttributes = []; + + $mediaFields = [ + 'id', + 'path', + 'mtime', + 'length', + 'title', + 'artist', + 'album', + 'genre', + 'isrc', ]; - foreach ($this->customFieldRepo->getFieldIds() as $fieldId => $fieldShortCode) { - $indexSettings['sortableAttributes'][] = 'custom_field_' . $fieldId; + foreach ($this->iterateStations() as $station) { + $stationId = $station->getIdRequired(); + + $filterableAttributes[] = 'station_' . $stationId . '_playlists'; + $filterableAttributes[] = 'station_' . $stationId . '_is_requestable'; + $filterableAttributes[] = 'station_' . $stationId . '_is_on_demand'; } - $this->client->updateIndex( - $this->indexUid, - $indexSettings + foreach ($this->customFieldRepo->getFieldIds() as $fieldId => $fieldShortCode) { + $mediaFields[] = 'custom_field_' . $fieldId; + } + + $indexSettings = [ + 'primaryKey' => 'id', + 'filterableAttributes' => $filterableAttributes, + 'sortableAttributes' => $mediaFields, + 'displayedAttributes' => $this->environment->isProduction() + ? ['id'] + : ['*'], + ]; + + // Avoid updating settings unless necessary to avoid triggering a reindex. + $this->client->createIndex($this->indexUid); + + $index = $this->client->index($this->indexUid); + $currentSettings = $index->getSettings(); + $settingsToUpdate = []; + + foreach ($indexSettings as $settingKey => $setting) { + $currentSetting = $currentSettings[$settingKey] ?? []; + if ($currentSetting !== $setting) { + $settingsToUpdate[$settingKey] = $setting; + } + } + + if (!empty($settingsToUpdate)) { + $index->updateSettings($settingsToUpdate); + } + } + + public function addMedia(array $ids): void + { + $this->refreshMedia($ids, true); + } + + public function refreshMedia( + array $ids, + bool $includePlaylists = false + ): void { + } + + public function refreshPlaylists(Station $station): void + { + $stationId = $station->getIdRequired(); + + $playlistsKey = 'station_' . $stationId . '_playlists'; + $isRequestableKey = 'station_' . $stationId . '_is_requestable'; + $isOnDemandKey = 'station_' . $stationId . '_is_on_demand'; + + $allMediaRaw = $this->em->createQuery( + <<<'DQL' + SELECT m.id, m.unique_id FROM App\Entity\StationMedia m + WHERE m.storage_location = :storageLocation + DQL + )->setParameter('storageLocation', $this->storageLocation) + ->getArrayResult(); + + $media = []; + foreach ($allMediaRaw as $mediaRow) { + $media[$mediaRow['id']] = [ + 'id' => $mediaRow['unique_id'], + $playlistsKey => [], + $isRequestableKey => false, + $isOnDemandKey => false, + ]; + } + + $allPlaylists = $this->em->createQuery( + <<<'DQL' + SELECT p.id, p.include_in_on_demand, p.include_in_requests + FROM App\Entity\StationPlaylist p + WHERE p.station = :station AND p.is_enabled = 1 + DQL + )->setParameter('station', $station) + ->getArrayResult(); + + $allPlaylistIds = []; + $onDemandPlaylists = []; + $requestablePlaylists = []; + + foreach ($allPlaylists as $playlist) { + $allPlaylistIds[] = $playlist['id']; + if ($playlist['include_in_on_demand']) { + $onDemandPlaylists[$playlist['id']] = $playlist['id']; + } + if ($playlist['include_in_requests']) { + $requestablePlaylists[$playlist['id']] = $playlist['id']; + } + } + + $mediaInPlaylists = $this->em->createQuery( + <<<'DQL' + SELECT spm.media_id, spm.playlist_id + FROM App\Entity\StationPlaylistMedia spm + WHERE spm.playlist_id IN (:allPlaylistIds) + DQL + )->setParameter('allPlaylistIds', $allPlaylistIds) + ->toIterable(); + + foreach ($mediaInPlaylists as $spmRow) { + $mediaId = $spmRow['media_id']; + $playlistId = $spmRow['playlist_id']; + + $media[$mediaId][$playlistsKey][] = $playlistId; + if (isset($requestablePlaylists[$playlistId])) { + $media[$mediaId][$isRequestableKey] = true; + } + if (isset($onDemandPlaylists[$playlistId])) { + $media[$mediaId][$isOnDemandKey] = true; + } + } + + $this->client->index($this->indexUid)->updateDocumentsInBatches( + array_values($media) ); } @@ -59,6 +173,7 @@ final class Index <<<'DQL' SELECT s FROM App\Entity\Station s WHERE s.media_storage_location = :storageLocation + AND s.is_enabled = 1 DQL )->setParameter('storageLocation', $this->storageLocation) ->toIterable(); From bb499874ca074d7bb92ce8a49623923a3ad55275 Mon Sep 17 00:00:00 2001 From: Buster Neece Date: Sat, 28 Jan 2023 08:13:02 -0600 Subject: [PATCH 05/11] More PHP work! --- config/events.php | 1 + src/Entity/StationMedia.php | 9 - src/Service/Meilisearch.php | 7 +- src/Service/Meilisearch/Index.php | 209 +++++++++++++++++++++-- src/Sync/Task/UpdateMeilisearchIndex.php | 114 +++++++++++++ 5 files changed, 311 insertions(+), 29 deletions(-) create mode 100644 src/Sync/Task/UpdateMeilisearchIndex.php diff --git a/config/events.php b/config/events.php index f1e7dd4fb..99f2f38a0 100644 --- a/config/events.php +++ b/config/events.php @@ -144,6 +144,7 @@ return static function (CallableEventDispatcherInterface $dispatcher) { App\Sync\Task\RunBackupTask::class, App\Sync\Task\SendTimeOnSocketTask::class, App\Sync\Task\UpdateGeoLiteTask::class, + App\Sync\Task\UpdateMeilisearchIndex::class, App\Sync\Task\UpdateStorageLocationSizesTask::class, ]); } diff --git a/src/Entity/StationMedia.php b/src/Entity/StationMedia.php index 738dd8c7d..f4476ed96 100644 --- a/src/Entity/StationMedia.php +++ b/src/Entity/StationMedia.php @@ -192,15 +192,6 @@ class StationMedia implements ] protected int $art_updated_at = 0; - #[ - OA\Property( - description: "The latest time (UNIX timestamp) when the search record for this entry was updated.", - example: OpenApi::SAMPLE_TIMESTAMP - ), - ORM\Column - ] - protected int $search_updated_at = 0; - /** @var Collection */ #[ OA\Property(type: "array", items: new OA\Items()), diff --git a/src/Service/Meilisearch.php b/src/Service/Meilisearch.php index b73d1904f..be89386f3 100644 --- a/src/Service/Meilisearch.php +++ b/src/Service/Meilisearch.php @@ -14,7 +14,7 @@ use Meilisearch\Client; final class Meilisearch { - public const BATCH_SIZE = 50; + public const BATCH_SIZE = 100; public function __construct( private readonly Environment $environment, @@ -52,12 +52,13 @@ final class Meilisearch public function getIndex(StorageLocation $storageLocation): Index { + $client = $this->getClient(); + return $this->factory->make( Index::class, [ 'storageLocation' => $storageLocation, - 'indexUid' => self::getIndexUid($storageLocation), - 'client' => $this->getClient(), + 'indexClient' => $client->index(self::getIndexUid($storageLocation)), ] ); } diff --git a/src/Service/Meilisearch/Index.php b/src/Service/Meilisearch/Index.php index 2a18dddfd..82dfacb51 100644 --- a/src/Service/Meilisearch/Index.php +++ b/src/Service/Meilisearch/Index.php @@ -9,7 +9,9 @@ use App\Entity\Repository\CustomFieldRepository; use App\Entity\Station; use App\Entity\StorageLocation; use App\Environment; -use Meilisearch\Client; +use App\Service\Meilisearch; +use Meilisearch\Contracts\DocumentsQuery; +use Meilisearch\Endpoints\Indexes; final class Index { @@ -17,9 +19,8 @@ final class Index private readonly ReloadableEntityManagerInterface $em, private readonly CustomFieldRepository $customFieldRepo, private readonly Environment $environment, - private readonly Client $client, private readonly StorageLocation $storageLocation, - private readonly string $indexUid + private readonly Indexes $indexClient, ) { } @@ -39,9 +40,7 @@ final class Index 'isrc', ]; - foreach ($this->iterateStations() as $station) { - $stationId = $station->getIdRequired(); - + foreach ($this->getStationIds() as $stationId) { $filterableAttributes[] = 'station_' . $stationId . '_playlists'; $filterableAttributes[] = 'station_' . $stationId . '_is_requestable'; $filterableAttributes[] = 'station_' . $stationId . '_is_on_demand'; @@ -52,7 +51,6 @@ final class Index } $indexSettings = [ - 'primaryKey' => 'id', 'filterableAttributes' => $filterableAttributes, 'sortableAttributes' => $mediaFields, 'displayedAttributes' => $this->environment->isProduction() @@ -61,10 +59,12 @@ final class Index ]; // Avoid updating settings unless necessary to avoid triggering a reindex. - $this->client->createIndex($this->indexUid); + $this->indexClient->create( + $this->indexClient->getUid(), + ['primaryKey' => 'id'] + ); - $index = $this->client->index($this->indexUid); - $currentSettings = $index->getSettings(); + $currentSettings = $this->indexClient->getSettings(); $settingsToUpdate = []; foreach ($indexSettings as $settingKey => $setting) { @@ -75,10 +75,42 @@ final class Index } if (!empty($settingsToUpdate)) { - $index->updateSettings($settingsToUpdate); + $this->indexClient->updateSettings($settingsToUpdate); } } + public function getIdsInIndex(): iterable + { + $perPage = Meilisearch::BATCH_SIZE; + $documentsQuery = (new DocumentsQuery()) + ->setOffset(0) + ->setLimit($perPage) + ->setFields(['id']); + + $documents = $this->indexClient->getDocuments($documentsQuery); + foreach ($documents->getIterator() as $document) { + yield $document['id']; + } + + if ($documents->getTotal() <= $perPage) { + return; + } + + $totalPages = ceil($documents->getTotal() / $perPage); + for ($page = 1; $page <= $totalPages; $page++) { + $documentsQuery->setOffset($page * $perPage); + $documents = $this->indexClient->getDocuments($documentsQuery); + foreach ($documents->getIterator() as $document) { + yield $document['id']; + } + } + } + + public function deleteIds(array $ids): void + { + $this->indexClient->deleteDocuments($ids); + } + public function addMedia(array $ids): void { $this->refreshMedia($ids, true); @@ -88,6 +120,148 @@ final class Index array $ids, bool $includePlaylists = false ): void { + if ($includePlaylists) { + $mediaPlaylistsRaw = $this->em->createQuery( + <<<'DQL' + SELECT spm.media_id, spm.playlist_id + FROM App\Entity\StationPlaylistMedia spm + WHERE spm.media_id IN (:mediaIds) + DQL + )->setParameter('mediaIds', $ids) + ->getArrayResult(); + + $mediaPlaylists = []; + $playlistIds = []; + + foreach ($mediaPlaylistsRaw as $mediaPlaylistRow) { + $mediaId = $mediaPlaylistRow['media_id']; + $playlistId = $mediaPlaylistRow['playlist_id']; + + $playlistIds[$playlistId] = $playlistId; + + $mediaPlaylists[$mediaId] ??= []; + $mediaPlaylists[$mediaId][] = $playlistId; + } + + $stationIds = $this->getStationIds(); + + $playlistsRaw = $this->em->createQuery( + <<<'DQL' + SELECT p.id, p.station_id, p.include_in_on_demand, p.include_in_requests + FROM App\Entity\StationPlaylist p + WHERE p.id IN (:playlistIds) AND p.station_id IN (:stationIds) + AND p.is_enabled = 1 + DQL + )->setParameter('playlistIds', $playlistIds) + ->setParameter('stationIds', $stationIds) + ->getArrayResult(); + + $playlists = []; + foreach ($playlistsRaw as $playlistRow) { + $playlists[$playlistRow['id']] = $playlistRow; + } + } + + $customFieldsRaw = $this->em->createQuery( + <<<'DQL' + SELECT smcf.media_id, smcf.field_id, smcf.value + FROM App\Entity\StationMediaCustomField smcf + WHERE smcf.media_id IN (:mediaIds) + DQL + )->setParameter('mediaIds', $ids) + ->getArrayResult(); + + $customFields = []; + foreach ($customFieldsRaw as $customFieldRow) { + $mediaId = $customFieldRow['media_id']; + + $customFields[$mediaId] ??= []; + $customFields[$mediaId]['custom_field_' . $customFieldRow['field_id']] = $customFieldRow['value']; + } + + $mediaRaw = $this->em->createQuery( + <<<'DQL' + SELECT sm.id, + sm.unique_id, + sm.path, + sm.mtime, + sm.length_text, + sm.title, + sm.artist, + sm.album, + sm.genre, + sm.isrc + FROM App\Entity\StationMedia sm + WHERE sm.storage_location = :storageLocation + AND sm.id IN (:ids) + DQL + )->setParameter('storageLocation', $this->storageLocation) + ->setParameter('ids', $ids) + ->toIterable(); + + $media = []; + + foreach ($mediaRaw as $row) { + $mediaId = $row['id']; + + $record = [ + 'id' => $row['unique_id'], + 'path' => $row['path'], + 'mtime' => $row['mtime'], + 'duration' => $row['length_text'], + 'title' => $row['title'], + 'artist' => $row['artist'], + 'album' => $row['album'], + 'genre' => $row['genre'], + 'isrc' => $row['isrc'], + ]; + + if (isset($customFields[$mediaId])) { + $record = array_merge($record, $customFields[$mediaId]); + } + + if ($includePlaylists) { + foreach ($stationIds as $stationId) { + $record['station_' . $stationId . '_playlists'] = []; + $record['station_' . $stationId . '_is_requestable'] = false; + $record['station_' . $stationId . '_is_on_demand'] = false; + } + + if (isset($mediaPlaylists[$mediaId])) { + foreach ($mediaPlaylists[$mediaId] as $mediaPlaylistId) { + if (!isset($playlists[$mediaPlaylistId])) { + continue; + } + + $playlist = $playlists[$mediaPlaylistId]; + $stationId = $playlist['station_id']; + + $record['station_' . $stationId . '_playlists'][] = $mediaPlaylistId; + + if ($playlist['include_in_requests']) { + $record['station_' . $stationId . '_is_requestable'] = true; + } + if ($playlist['include_in_on_demand']) { + $record['station_' . $stationId . '_is_on_demand'] = true; + } + } + } + } + + $media[$mediaId] = $record; + } + + if ($includePlaylists) { + $this->indexClient->addDocumentsInBatches( + $media, + Meilisearch::BATCH_SIZE + ); + } else { + $this->indexClient->updateDocumentsInBatches( + $media, + Meilisearch::BATCH_SIZE + ); + } } public function refreshPlaylists(Station $station): void @@ -161,21 +335,22 @@ final class Index } } - $this->client->index($this->indexUid)->updateDocumentsInBatches( - array_values($media) + $this->indexClient->updateDocumentsInBatches( + array_values($media), + Meilisearch::BATCH_SIZE ); } - /** @return Station[] */ - private function iterateStations(): iterable + /** @return int[] */ + private function getStationIds(): array { return $this->em->createQuery( <<<'DQL' - SELECT s FROM App\Entity\Station s + SELECT s.id FROM App\Entity\Station s WHERE s.media_storage_location = :storageLocation AND s.is_enabled = 1 DQL )->setParameter('storageLocation', $this->storageLocation) - ->toIterable(); + ->getSingleColumnResult(); } } diff --git a/src/Sync/Task/UpdateMeilisearchIndex.php b/src/Sync/Task/UpdateMeilisearchIndex.php new file mode 100644 index 000000000..25d9e29fd --- /dev/null +++ b/src/Sync/Task/UpdateMeilisearchIndex.php @@ -0,0 +1,114 @@ +meilisearch->isSupported()) { + $this->logger->debug('Meilisearch is not supported on this instance. Skipping sync task.'); + } + + $storageLocations = $this->iterateStorageLocations(Entity\Enums\StorageLocationTypes::StationMedia); + + foreach ($storageLocations as $storageLocation) { + $this->logger->info( + sprintf( + 'Updating MeiliSearch index for storage location %s...', + $storageLocation + ) + ); + + $this->updateIndex($storageLocation); + } + } + + public function updateIndex(Entity\StorageLocation $storageLocation): void + { + $index = $this->meilisearch->getIndex($storageLocation); + $index->configure(); + + $existingIdsRaw = iterator_to_array($index->getIdsInIndex(), false); + $existingIds = array_combine($existingIdsRaw, $existingIdsRaw); + + $queuedMedia = []; + + foreach ( + $this->queueManager->getMessagesInTransport( + QueueManagerInterface::QUEUE_NORMAL_PRIORITY + ) as $message + ) { + if ($message instanceof AddMediaToSearchIndexMessage) { + foreach ($message->media as $mediaId) { + $queuedMedia[$mediaId] = $mediaId; + } + } + } + + $mediaRaw = $this->em->createQuery( + <<<'DQL' + SELECT sm.id, sm.unique_id + FROM App\Entity\StationMedia sm + WHERE sm.storage_location = :storageLocation + DQL + )->setParameter('storageLocation', $storageLocation) + ->toIterable([], AbstractQuery::HYDRATE_ARRAY); + + $newIds = []; + + foreach ($mediaRaw as $row) { + if ( + isset($existingIds[$row['unique_id']]) + || isset($queuedMedia[$row['id']]) + ) { + unset($existingIds[$row['unique_id']]); + continue; + } + + $newIds[] = $row['id']; + } + + foreach (array_chunk($newIds, Meilisearch::BATCH_SIZE) as $batchIds) { + $message = new AddMediaToSearchIndexMessage(); + $message->storage_location_id = $storageLocation->getIdRequired(); + $message->media = $batchIds; + $this->messageBus->dispatch($message); + } + + if (!empty($existingIds)) { + $index->deleteIds($existingIds); + } + } +} From 81d16d619a57b622b0866c1e0d5738f0e84cc938 Mon Sep 17 00:00:00 2001 From: Buster Neece Date: Sun, 29 Jan 2023 08:15:26 -0600 Subject: [PATCH 06/11] More Meilisearch infrastructure work. --- config/messagequeue.php | 3 + config/services.php | 1 + docker-compose.sample.yml | 2 - frontend/vue/components/Public/OnDemand.vue | 38 +++- .../Api/Stations/Files/BatchAction.php | 25 ++- .../Api/Stations/FilesController.php | 18 ++ .../Api/Stations/OnDemand/ListAction.php | 129 ++++------- .../Api/Stations/PlaylistsController.php | 51 +++++ .../Frontend/PublicPages/OnDemandAction.php | 2 +- src/Media/BatchUtilities.php | 52 ++++- src/Message/AddMediaToSearchIndexMessage.php | 33 --- src/Message/Meilisearch/AddMediaMessage.php | 19 ++ .../Meilisearch/UpdatePlaylistsMessage.php | 16 ++ src/Paginator.php | 37 ++-- src/Service/Meilisearch.php | 8 +- src/Service/Meilisearch/Index.php | 209 +++++++++++++----- src/Service/Meilisearch/MessageHandler.php | 65 ++++++ src/Service/Meilisearch/PaginatorAdapter.php | 60 +++++ src/Sync/Task/CheckFolderPlaylistsTask.php | 16 +- src/Sync/Task/UpdateMeilisearchIndex.php | 62 ++++-- 20 files changed, 620 insertions(+), 226 deletions(-) delete mode 100644 src/Message/AddMediaToSearchIndexMessage.php create mode 100644 src/Message/Meilisearch/AddMediaMessage.php create mode 100644 src/Message/Meilisearch/UpdatePlaylistsMessage.php create mode 100644 src/Service/Meilisearch/MessageHandler.php create mode 100644 src/Service/Meilisearch/PaginatorAdapter.php diff --git a/config/messagequeue.php b/config/messagequeue.php index eec78b259..a4924bcea 100644 --- a/config/messagequeue.php +++ b/config/messagequeue.php @@ -20,5 +20,8 @@ return [ Message\DispatchWebhookMessage::class => App\Webhook\Dispatcher::class, Message\TestWebhookMessage::class => App\Webhook\Dispatcher::class, + Message\Meilisearch\AddMediaMessage::class => App\Service\Meilisearch\MessageHandler::class, + Message\Meilisearch\UpdatePlaylistsMessage::class => App\Service\Meilisearch\MessageHandler::class, + Mailer\Messenger\SendEmailMessage::class => Mailer\Messenger\MessageHandler::class, ]; diff --git a/config/services.php b/config/services.php index 0e7b63a1d..899b2fbfa 100644 --- a/config/services.php +++ b/config/services.php @@ -125,6 +125,7 @@ return [ // $config->setSQLLogger(new Doctrine\DBAL\Logging\EchoSQLLogger); $config->addCustomNumericFunction('RAND', DoctrineExtensions\Query\Mysql\Rand::class); + $config->addCustomStringFunction('FIELD', DoctrineExtensions\Query\Mysql\Field::class); if (!Doctrine\DBAL\Types\Type::hasType('carbon_immutable')) { Doctrine\DBAL\Types\Type::addType('carbon_immutable', Carbon\Doctrine\CarbonImmutableType::class); diff --git a/docker-compose.sample.yml b/docker-compose.sample.yml index 16d0dc4dc..c5e10c068 100644 --- a/docker-compose.sample.yml +++ b/docker-compose.sample.yml @@ -191,7 +191,6 @@ services: - stereo_tool_install:/var/azuracast/servers/stereo_tool - geolite_install:/var/azuracast/geoip - sftpgo_data:/var/azuracast/sftpgo/persist - - meilisearch_data:/var/azuracast/meilisearch/persist - backups:/var/azuracast/backups - acme:/var/azuracast/acme - db_data:/var/lib/mysql @@ -222,7 +221,6 @@ volumes: stereo_tool_install: { } geolite_install: { } sftpgo_data: { } - meilisearch_data: { } station_data: { } www_uploads: { } backups: { } diff --git a/frontend/vue/components/Public/OnDemand.vue b/frontend/vue/components/Public/OnDemand.vue index f7d3e53db..7c739d1b0 100644 --- a/frontend/vue/components/Public/OnDemand.vue +++ b/frontend/vue/components/Public/OnDemand.vue @@ -58,9 +58,9 @@ -