Compare commits
5 Commits
be254f577c
...
4bad6be54c
Author | SHA1 | Date |
---|---|---|
ideclon | 4bad6be54c | |
ideclon | 9e8f450cd4 | |
ideclon | afa37c99fb | |
ideclon | 3153b6faa5 | |
ideclon | be76f3f79c |
|
@ -0,0 +1,42 @@
|
|||
# Mastodon Algorithm - Server Components
|
||||
|
||||
This repo contains server components for [ideclon's](https://ideclon.uk) Mastodon Algorithm project.
|
||||
|
||||
**This project relies on [AppWrite](https://appwrite.io). You must have an AppWrite instance to run this project**
|
||||
|
||||
This repo contains two parts:
|
||||
* `functions`
|
||||
* `collections` (aka databases)
|
||||
|
||||
Once the server components are deployed, you can deploy the frontend.
|
||||
|
||||
# Deploy
|
||||
To deploy this repo, you'll need the [AppWrite CLI](https://appwrite.io/docs/command-line#installation).
|
||||
|
||||
## Connect to your Appwrite instance
|
||||
Log into your Appwrite instance's UI and create a new project. Call it whatever you want. Once it's created, navigate to Settings within the new project.
|
||||
|
||||
From the root of this repo, run `appwrite client --endpoint [YOUR_APPWRITE_API_ENDPOINT] --projectId [PROJECT_ID]`.
|
||||
|
||||
Next, run `appwrite login` to authenticate.
|
||||
|
||||
Now, link the repo to the project - `appwrite init project`. Select yes when ask if you want to override the currently associated project. Select "List this directory to an existing Appwrite project", then select the new project you've just created.
|
||||
|
||||
## Deploy to AppWrite
|
||||
Run `appwrite deploy collection` to deploy the database. Press `<a>` to select all, then `<enter>` to proceed.
|
||||
|
||||
Once that's done, run `appwrite deploy function` to deploy the backend functions, and proceed as above.
|
||||
|
||||
## Generate API key
|
||||
Navigate to Overview and select "API Key" under "Integrate With Your Server". Give the key a name, "Select All" for scopes, and click "Create". Scroll down to Integrations > API Keys, click the new key you just created, and copy it (under "API Key Secret").
|
||||
|
||||
## Set enviroment vars
|
||||
This currently has to be done manually for each function. There's an [issue](https://github.com/appwrite/appwrite/issues/3530) open at Appwrite to make this possible on a project level.
|
||||
|
||||
Navigate to the first function > Settings. Scroll down to Update Function Variables. Set the following vars:
|
||||
* `APPWRITE_FUNCTION_ENDPOINT`: `http://172.17.0.1:8080/v1`
|
||||
* `APPWRITE_DATABASE_ID`: This is the database ID of the database created above. Get it at Databases > click "Project ID" on the database created above.
|
||||
* `BASE_URI`: This is the URL of the frontend
|
||||
* `API_KEY`: The Appwrite API key you created above
|
||||
|
||||
You'll need to create these vars for each of the three functions.
|
331
appwrite.json
331
appwrite.json
|
@ -45,8 +45,337 @@
|
|||
],
|
||||
"execute": [],
|
||||
"events": [],
|
||||
"schedule": "",
|
||||
"schedule": "*/1 * * * *",
|
||||
"timeout": 15
|
||||
}
|
||||
],
|
||||
"collections": [
|
||||
{
|
||||
"$id": "servers",
|
||||
"$createdAt": "2022-11-28T00:15:46.337+00:00",
|
||||
"$updatedAt": "2022-11-28T00:16:55.140+00:00",
|
||||
"$permissions": [
|
||||
"read(\"users\")"
|
||||
],
|
||||
"databaseId": "637fd77aae471453b595",
|
||||
"name": "servers",
|
||||
"enabled": true,
|
||||
"documentSecurity": false,
|
||||
"attributes": [
|
||||
{
|
||||
"key": "server_url",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 1000,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "client_id",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 100,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "client_secret",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 100,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "app_access_token",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": false,
|
||||
"array": false,
|
||||
"size": 100,
|
||||
"default": null
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"key": "server_url",
|
||||
"type": "unique",
|
||||
"status": "available",
|
||||
"attributes": [
|
||||
"server_url"
|
||||
],
|
||||
"orders": [
|
||||
"DESC"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"$id": "hashtags",
|
||||
"$createdAt": "2022-11-30T20:47:50.076+00:00",
|
||||
"$updatedAt": "2022-11-30T20:47:50.076+00:00",
|
||||
"$permissions": [],
|
||||
"databaseId": "637fd77aae471453b595",
|
||||
"name": "hashtags",
|
||||
"enabled": true,
|
||||
"documentSecurity": false,
|
||||
"attributes": [
|
||||
{
|
||||
"key": "user",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 100,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "hashtag",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 500,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "lastseen",
|
||||
"type": "datetime",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"format": "",
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "points",
|
||||
"type": "integer",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"min": 0,
|
||||
"max": 9223372036854775807,
|
||||
"default": null
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"key": "user_id",
|
||||
"type": "key",
|
||||
"status": "available",
|
||||
"attributes": [
|
||||
"user"
|
||||
],
|
||||
"orders": [
|
||||
"DESC"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "hashtag",
|
||||
"type": "key",
|
||||
"status": "available",
|
||||
"attributes": [
|
||||
"hashtag"
|
||||
],
|
||||
"orders": [
|
||||
"DESC"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "getUsersHashtags",
|
||||
"type": "key",
|
||||
"status": "available",
|
||||
"attributes": [
|
||||
"user",
|
||||
"hashtag"
|
||||
],
|
||||
"orders": [
|
||||
"DESC",
|
||||
"DESC"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"$id": "index_history",
|
||||
"$createdAt": "2022-11-30T21:05:31.728+00:00",
|
||||
"$updatedAt": "2022-11-30T21:05:31.728+00:00",
|
||||
"$permissions": [],
|
||||
"databaseId": "637fd77aae471453b595",
|
||||
"name": "index_history",
|
||||
"enabled": true,
|
||||
"documentSecurity": false,
|
||||
"attributes": [
|
||||
{
|
||||
"key": "user_id",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 100,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "last_post_id",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 300,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "populating_new_index",
|
||||
"type": "boolean",
|
||||
"status": "available",
|
||||
"required": false,
|
||||
"array": false,
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"key": "first_post_id",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": false,
|
||||
"array": false,
|
||||
"size": 500,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "feed_name",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"elements": [
|
||||
"home",
|
||||
"favourites",
|
||||
"bookmarks",
|
||||
"local",
|
||||
"federated",
|
||||
"profile"
|
||||
],
|
||||
"format": "enum",
|
||||
"default": null
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"key": "queryIndexHistory",
|
||||
"type": "key",
|
||||
"status": "available",
|
||||
"attributes": [
|
||||
"user_id",
|
||||
"feed_name"
|
||||
],
|
||||
"orders": [
|
||||
"DESC",
|
||||
"DESC"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "unique",
|
||||
"type": "unique",
|
||||
"status": "available",
|
||||
"attributes": [
|
||||
"user_id",
|
||||
"feed_name"
|
||||
],
|
||||
"orders": [
|
||||
"DESC",
|
||||
"DESC"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"$id": "suggested",
|
||||
"$createdAt": "2022-12-19T00:03:11.695+00:00",
|
||||
"$updatedAt": "2022-12-19T23:18:53.402+00:00",
|
||||
"$permissions": [],
|
||||
"databaseId": "637fd77aae471453b595",
|
||||
"name": "suggested",
|
||||
"enabled": true,
|
||||
"documentSecurity": true,
|
||||
"attributes": [
|
||||
{
|
||||
"key": "post_link",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 2000,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "points",
|
||||
"type": "integer",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"min": 0,
|
||||
"max": 9223372036854775807,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "user_id",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": true,
|
||||
"array": false,
|
||||
"size": 100,
|
||||
"default": null
|
||||
},
|
||||
{
|
||||
"key": "local_link",
|
||||
"type": "string",
|
||||
"status": "available",
|
||||
"required": false,
|
||||
"array": false,
|
||||
"size": 2000,
|
||||
"default": null
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"key": "point_sort",
|
||||
"type": "key",
|
||||
"status": "available",
|
||||
"attributes": [
|
||||
"points",
|
||||
"user_id"
|
||||
],
|
||||
"orders": [
|
||||
"DESC",
|
||||
"DESC"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "points",
|
||||
"type": "key",
|
||||
"status": "available",
|
||||
"attributes": [
|
||||
"points"
|
||||
],
|
||||
"orders": [
|
||||
"DESC"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "link_unique",
|
||||
"type": "unique",
|
||||
"status": "failed",
|
||||
"attributes": [
|
||||
"post_link"
|
||||
],
|
||||
"orders": [
|
||||
"DESC"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -28,8 +28,6 @@ require_once 'vendor/autoload.php';
|
|||
If an error is thrown, a response with code 500 will be returned.
|
||||
*/
|
||||
|
||||
// require_once 'threading/ThreadQueue.php';
|
||||
|
||||
include "crawl.php";
|
||||
|
||||
return function($req, $res) {
|
||||
|
|
|
@ -9,10 +9,95 @@ use Appwrite\Query;
|
|||
|
||||
class Suggestions {
|
||||
public static function crawlFederatedFeed($user, $client) {
|
||||
|
||||
$request = new \GuzzleHttp\Client(['base_uri' => $user['prefs']['user_server_uri']]);
|
||||
|
||||
try {
|
||||
$response = $request->request('GET', "/api/v1/timelines/public", [
|
||||
'headers' => [
|
||||
'Authorization' => 'Bearer ' . $user['prefs']['user_token']
|
||||
]
|
||||
]);
|
||||
|
||||
$responseBody = (string)$response->getBody();
|
||||
$responseBody = json_decode($responseBody);
|
||||
|
||||
$statuses = [];
|
||||
|
||||
foreach($responseBody as $status) {
|
||||
if(count($status->tags) == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
public static function scoreStatus($status, $user_id, $client) {
|
||||
$points = self::scoreStatus($status, $user, $client);
|
||||
|
||||
if($points['points'] !== 0) {
|
||||
$statuses[] = $points;
|
||||
}
|
||||
}
|
||||
|
||||
usort($statuses, function ($a, $b) {
|
||||
var_dump($a);
|
||||
// return 0 <=> 0;
|
||||
return $b['points'] <=> $a['points'];
|
||||
});
|
||||
|
||||
$database = new Databases($client);
|
||||
|
||||
foreach($statuses as $status) {
|
||||
$response = $request->request('GET', "/api/v2/search?type=statuses&resolve=true&q=" . $status['post'], [
|
||||
'headers' => [
|
||||
'Authorization' => 'Bearer ' . $user['prefs']['user_token']
|
||||
]
|
||||
]);
|
||||
|
||||
$responseBody = (string)$response->getBody();
|
||||
$responseBody = json_decode($responseBody);
|
||||
|
||||
$result = $responseBody->statuses;
|
||||
|
||||
if(isset($result[0])) {
|
||||
$result = $result[0];
|
||||
var_dump($result);
|
||||
$database->createDocument(getenv('APPWRITE_DATABASE_ID'), 'suggested', 'unique()', [
|
||||
'post_link' => $status['post'],
|
||||
'points' => $status['points'],
|
||||
'local_link' => $user['prefs']['user_server_uri'] . "/@" . $result->account->acct . "/" . $result->id,
|
||||
'user_id' => $user['$id']
|
||||
], [
|
||||
\Appwrite\Permission::read(\Appwrite\Role::user($user['$id']))
|
||||
]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return $statuses;
|
||||
|
||||
} catch (\GuzzleHttp\Exception\ClientException $e) {
|
||||
return [$e->getMessage()];
|
||||
}
|
||||
}
|
||||
|
||||
public static function scoreStatus($status, $user, $client) {
|
||||
$database = new Databases($client);
|
||||
$query = new Query();
|
||||
|
||||
$points = 0;
|
||||
|
||||
foreach($status->tags as $tag) {
|
||||
$record = $database->listDocuments(getenv('APPWRITE_DATABASE_ID'), 'hashtags', [
|
||||
$query->equal('user', $user['$id']),
|
||||
$query->equal('hashtag', $tag->name)
|
||||
])['documents'];
|
||||
|
||||
if(isset($record[0])) {
|
||||
$points += $record[0]['points'];
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
'post' => $status->url,
|
||||
'points' => $points
|
||||
];
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue