Compare commits
No commits in common. "4bad6be54c6977a448d83084f5c3c1e22f6ae08a" and "be254f577ccb7fb57e3dba5c8494b0082ef5ecdc" have entirely different histories.
4bad6be54c
...
be254f577c
42
README.md
42
README.md
|
@ -1,42 +0,0 @@
|
||||||
# Mastodon Algorithm - Server Components
|
|
||||||
|
|
||||||
This repo contains server components for [ideclon's](https://ideclon.uk) Mastodon Algorithm project.
|
|
||||||
|
|
||||||
**This project relies on [AppWrite](https://appwrite.io). You must have an AppWrite instance to run this project**
|
|
||||||
|
|
||||||
This repo contains two parts:
|
|
||||||
* `functions`
|
|
||||||
* `collections` (aka databases)
|
|
||||||
|
|
||||||
Once the server components are deployed, you can deploy the frontend.
|
|
||||||
|
|
||||||
# Deploy
|
|
||||||
To deploy this repo, you'll need the [AppWrite CLI](https://appwrite.io/docs/command-line#installation).
|
|
||||||
|
|
||||||
## Connect to your Appwrite instance
|
|
||||||
Log into your Appwrite instance's UI and create a new project. Call it whatever you want. Once it's created, navigate to Settings within the new project.
|
|
||||||
|
|
||||||
From the root of this repo, run `appwrite client --endpoint [YOUR_APPWRITE_API_ENDPOINT] --projectId [PROJECT_ID]`.
|
|
||||||
|
|
||||||
Next, run `appwrite login` to authenticate.
|
|
||||||
|
|
||||||
Now, link the repo to the project - `appwrite init project`. Select yes when ask if you want to override the currently associated project. Select "List this directory to an existing Appwrite project", then select the new project you've just created.
|
|
||||||
|
|
||||||
## Deploy to AppWrite
|
|
||||||
Run `appwrite deploy collection` to deploy the database. Press `<a>` to select all, then `<enter>` to proceed.
|
|
||||||
|
|
||||||
Once that's done, run `appwrite deploy function` to deploy the backend functions, and proceed as above.
|
|
||||||
|
|
||||||
## Generate API key
|
|
||||||
Navigate to Overview and select "API Key" under "Integrate With Your Server". Give the key a name, "Select All" for scopes, and click "Create". Scroll down to Integrations > API Keys, click the new key you just created, and copy it (under "API Key Secret").
|
|
||||||
|
|
||||||
## Set enviroment vars
|
|
||||||
This currently has to be done manually for each function. There's an [issue](https://github.com/appwrite/appwrite/issues/3530) open at Appwrite to make this possible on a project level.
|
|
||||||
|
|
||||||
Navigate to the first function > Settings. Scroll down to Update Function Variables. Set the following vars:
|
|
||||||
* `APPWRITE_FUNCTION_ENDPOINT`: `http://172.17.0.1:8080/v1`
|
|
||||||
* `APPWRITE_DATABASE_ID`: This is the database ID of the database created above. Get it at Databases > click "Project ID" on the database created above.
|
|
||||||
* `BASE_URI`: This is the URL of the frontend
|
|
||||||
* `API_KEY`: The Appwrite API key you created above
|
|
||||||
|
|
||||||
You'll need to create these vars for each of the three functions.
|
|
331
appwrite.json
331
appwrite.json
|
@ -45,337 +45,8 @@
|
||||||
],
|
],
|
||||||
"execute": [],
|
"execute": [],
|
||||||
"events": [],
|
"events": [],
|
||||||
"schedule": "*/1 * * * *",
|
"schedule": "",
|
||||||
"timeout": 15
|
"timeout": 15
|
||||||
}
|
}
|
||||||
],
|
|
||||||
"collections": [
|
|
||||||
{
|
|
||||||
"$id": "servers",
|
|
||||||
"$createdAt": "2022-11-28T00:15:46.337+00:00",
|
|
||||||
"$updatedAt": "2022-11-28T00:16:55.140+00:00",
|
|
||||||
"$permissions": [
|
|
||||||
"read(\"users\")"
|
|
||||||
],
|
|
||||||
"databaseId": "637fd77aae471453b595",
|
|
||||||
"name": "servers",
|
|
||||||
"enabled": true,
|
|
||||||
"documentSecurity": false,
|
|
||||||
"attributes": [
|
|
||||||
{
|
|
||||||
"key": "server_url",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 1000,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "client_id",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 100,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "client_secret",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 100,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "app_access_token",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": false,
|
|
||||||
"array": false,
|
|
||||||
"size": 100,
|
|
||||||
"default": null
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"indexes": [
|
|
||||||
{
|
|
||||||
"key": "server_url",
|
|
||||||
"type": "unique",
|
|
||||||
"status": "available",
|
|
||||||
"attributes": [
|
|
||||||
"server_url"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"$id": "hashtags",
|
|
||||||
"$createdAt": "2022-11-30T20:47:50.076+00:00",
|
|
||||||
"$updatedAt": "2022-11-30T20:47:50.076+00:00",
|
|
||||||
"$permissions": [],
|
|
||||||
"databaseId": "637fd77aae471453b595",
|
|
||||||
"name": "hashtags",
|
|
||||||
"enabled": true,
|
|
||||||
"documentSecurity": false,
|
|
||||||
"attributes": [
|
|
||||||
{
|
|
||||||
"key": "user",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 100,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "hashtag",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 500,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "lastseen",
|
|
||||||
"type": "datetime",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"format": "",
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "points",
|
|
||||||
"type": "integer",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"min": 0,
|
|
||||||
"max": 9223372036854775807,
|
|
||||||
"default": null
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"indexes": [
|
|
||||||
{
|
|
||||||
"key": "user_id",
|
|
||||||
"type": "key",
|
|
||||||
"status": "available",
|
|
||||||
"attributes": [
|
|
||||||
"user"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "hashtag",
|
|
||||||
"type": "key",
|
|
||||||
"status": "available",
|
|
||||||
"attributes": [
|
|
||||||
"hashtag"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "getUsersHashtags",
|
|
||||||
"type": "key",
|
|
||||||
"status": "available",
|
|
||||||
"attributes": [
|
|
||||||
"user",
|
|
||||||
"hashtag"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC",
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"$id": "index_history",
|
|
||||||
"$createdAt": "2022-11-30T21:05:31.728+00:00",
|
|
||||||
"$updatedAt": "2022-11-30T21:05:31.728+00:00",
|
|
||||||
"$permissions": [],
|
|
||||||
"databaseId": "637fd77aae471453b595",
|
|
||||||
"name": "index_history",
|
|
||||||
"enabled": true,
|
|
||||||
"documentSecurity": false,
|
|
||||||
"attributes": [
|
|
||||||
{
|
|
||||||
"key": "user_id",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 100,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "last_post_id",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 300,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "populating_new_index",
|
|
||||||
"type": "boolean",
|
|
||||||
"status": "available",
|
|
||||||
"required": false,
|
|
||||||
"array": false,
|
|
||||||
"default": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "first_post_id",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": false,
|
|
||||||
"array": false,
|
|
||||||
"size": 500,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "feed_name",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"elements": [
|
|
||||||
"home",
|
|
||||||
"favourites",
|
|
||||||
"bookmarks",
|
|
||||||
"local",
|
|
||||||
"federated",
|
|
||||||
"profile"
|
|
||||||
],
|
|
||||||
"format": "enum",
|
|
||||||
"default": null
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"indexes": [
|
|
||||||
{
|
|
||||||
"key": "queryIndexHistory",
|
|
||||||
"type": "key",
|
|
||||||
"status": "available",
|
|
||||||
"attributes": [
|
|
||||||
"user_id",
|
|
||||||
"feed_name"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC",
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "unique",
|
|
||||||
"type": "unique",
|
|
||||||
"status": "available",
|
|
||||||
"attributes": [
|
|
||||||
"user_id",
|
|
||||||
"feed_name"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC",
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"$id": "suggested",
|
|
||||||
"$createdAt": "2022-12-19T00:03:11.695+00:00",
|
|
||||||
"$updatedAt": "2022-12-19T23:18:53.402+00:00",
|
|
||||||
"$permissions": [],
|
|
||||||
"databaseId": "637fd77aae471453b595",
|
|
||||||
"name": "suggested",
|
|
||||||
"enabled": true,
|
|
||||||
"documentSecurity": true,
|
|
||||||
"attributes": [
|
|
||||||
{
|
|
||||||
"key": "post_link",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 2000,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "points",
|
|
||||||
"type": "integer",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"min": 0,
|
|
||||||
"max": 9223372036854775807,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "user_id",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": true,
|
|
||||||
"array": false,
|
|
||||||
"size": 100,
|
|
||||||
"default": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "local_link",
|
|
||||||
"type": "string",
|
|
||||||
"status": "available",
|
|
||||||
"required": false,
|
|
||||||
"array": false,
|
|
||||||
"size": 2000,
|
|
||||||
"default": null
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"indexes": [
|
|
||||||
{
|
|
||||||
"key": "point_sort",
|
|
||||||
"type": "key",
|
|
||||||
"status": "available",
|
|
||||||
"attributes": [
|
|
||||||
"points",
|
|
||||||
"user_id"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC",
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "points",
|
|
||||||
"type": "key",
|
|
||||||
"status": "available",
|
|
||||||
"attributes": [
|
|
||||||
"points"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"key": "link_unique",
|
|
||||||
"type": "unique",
|
|
||||||
"status": "failed",
|
|
||||||
"attributes": [
|
|
||||||
"post_link"
|
|
||||||
],
|
|
||||||
"orders": [
|
|
||||||
"DESC"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
]
|
||||||
}
|
}
|
|
@ -28,6 +28,8 @@ require_once 'vendor/autoload.php';
|
||||||
If an error is thrown, a response with code 500 will be returned.
|
If an error is thrown, a response with code 500 will be returned.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// require_once 'threading/ThreadQueue.php';
|
||||||
|
|
||||||
include "crawl.php";
|
include "crawl.php";
|
||||||
|
|
||||||
return function($req, $res) {
|
return function($req, $res) {
|
||||||
|
|
|
@ -9,95 +9,10 @@ use Appwrite\Query;
|
||||||
|
|
||||||
class Suggestions {
|
class Suggestions {
|
||||||
public static function crawlFederatedFeed($user, $client) {
|
public static function crawlFederatedFeed($user, $client) {
|
||||||
|
|
||||||
$request = new \GuzzleHttp\Client(['base_uri' => $user['prefs']['user_server_uri']]);
|
$request = new \GuzzleHttp\Client(['base_uri' => $user['prefs']['user_server_uri']]);
|
||||||
|
|
||||||
try {
|
|
||||||
$response = $request->request('GET', "/api/v1/timelines/public", [
|
|
||||||
'headers' => [
|
|
||||||
'Authorization' => 'Bearer ' . $user['prefs']['user_token']
|
|
||||||
]
|
|
||||||
]);
|
|
||||||
|
|
||||||
$responseBody = (string)$response->getBody();
|
|
||||||
$responseBody = json_decode($responseBody);
|
|
||||||
|
|
||||||
$statuses = [];
|
|
||||||
|
|
||||||
foreach($responseBody as $status) {
|
|
||||||
if(count($status->tags) == 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$points = self::scoreStatus($status, $user, $client);
|
|
||||||
|
|
||||||
if($points['points'] !== 0) {
|
|
||||||
$statuses[] = $points;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
usort($statuses, function ($a, $b) {
|
|
||||||
var_dump($a);
|
|
||||||
// return 0 <=> 0;
|
|
||||||
return $b['points'] <=> $a['points'];
|
|
||||||
});
|
|
||||||
|
|
||||||
$database = new Databases($client);
|
|
||||||
|
|
||||||
foreach($statuses as $status) {
|
|
||||||
$response = $request->request('GET', "/api/v2/search?type=statuses&resolve=true&q=" . $status['post'], [
|
|
||||||
'headers' => [
|
|
||||||
'Authorization' => 'Bearer ' . $user['prefs']['user_token']
|
|
||||||
]
|
|
||||||
]);
|
|
||||||
|
|
||||||
$responseBody = (string)$response->getBody();
|
|
||||||
$responseBody = json_decode($responseBody);
|
|
||||||
|
|
||||||
$result = $responseBody->statuses;
|
|
||||||
|
|
||||||
if(isset($result[0])) {
|
|
||||||
$result = $result[0];
|
|
||||||
var_dump($result);
|
|
||||||
$database->createDocument(getenv('APPWRITE_DATABASE_ID'), 'suggested', 'unique()', [
|
|
||||||
'post_link' => $status['post'],
|
|
||||||
'points' => $status['points'],
|
|
||||||
'local_link' => $user['prefs']['user_server_uri'] . "/@" . $result->account->acct . "/" . $result->id,
|
|
||||||
'user_id' => $user['$id']
|
|
||||||
], [
|
|
||||||
\Appwrite\Permission::read(\Appwrite\Role::user($user['$id']))
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
return $statuses;
|
|
||||||
|
|
||||||
} catch (\GuzzleHttp\Exception\ClientException $e) {
|
|
||||||
return [$e->getMessage()];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function scoreStatus($status, $user, $client) {
|
public static function scoreStatus($status, $user_id, $client) {
|
||||||
$database = new Databases($client);
|
|
||||||
$query = new Query();
|
|
||||||
|
|
||||||
$points = 0;
|
|
||||||
|
|
||||||
foreach($status->tags as $tag) {
|
|
||||||
$record = $database->listDocuments(getenv('APPWRITE_DATABASE_ID'), 'hashtags', [
|
|
||||||
$query->equal('user', $user['$id']),
|
|
||||||
$query->equal('hashtag', $tag->name)
|
|
||||||
])['documents'];
|
|
||||||
|
|
||||||
if(isset($record[0])) {
|
|
||||||
$points += $record[0]['points'];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return [
|
|
||||||
'post' => $status->url,
|
|
||||||
'points' => $points
|
|
||||||
];
|
|
||||||
}
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue