Compare commits

...

5 Commits

Author SHA1 Message Date
ideclon 4bad6be54c
README 2022-12-22 23:48:04 +00:00
ideclon 9e8f450cd4
add database 2022-12-22 23:47:59 +00:00
ideclon afa37c99fb
status links local to your homeserver 2022-12-20 02:40:54 +00:00
ideclon 3153b6faa5
add suggestions to db 2022-12-19 03:45:17 +00:00
ideclon be76f3f79c
calculate points 2022-12-19 03:22:48 +00:00
4 changed files with 458 additions and 4 deletions

42
README.md Normal file
View File

@ -0,0 +1,42 @@
# Mastodon Algorithm - Server Components
This repo contains server components for [ideclon's](https://ideclon.uk) Mastodon Algorithm project.
**This project relies on [AppWrite](https://appwrite.io). You must have an AppWrite instance to run this project**
This repo contains two parts:
* `functions`
* `collections` (aka databases)
Once the server components are deployed, you can deploy the frontend.
# Deploy
To deploy this repo, you'll need the [AppWrite CLI](https://appwrite.io/docs/command-line#installation).
## Connect to your Appwrite instance
Log into your Appwrite instance's UI and create a new project. Call it whatever you want. Once it's created, navigate to Settings within the new project.
From the root of this repo, run `appwrite client --endpoint [YOUR_APPWRITE_API_ENDPOINT] --projectId [PROJECT_ID]`.
Next, run `appwrite login` to authenticate.
Now, link the repo to the project - `appwrite init project`. Select yes when ask if you want to override the currently associated project. Select "List this directory to an existing Appwrite project", then select the new project you've just created.
## Deploy to AppWrite
Run `appwrite deploy collection` to deploy the database. Press `<a>` to select all, then `<enter>` to proceed.
Once that's done, run `appwrite deploy function` to deploy the backend functions, and proceed as above.
## Generate API key
Navigate to Overview and select "API Key" under "Integrate With Your Server". Give the key a name, "Select All" for scopes, and click "Create". Scroll down to Integrations > API Keys, click the new key you just created, and copy it (under "API Key Secret").
## Set enviroment vars
This currently has to be done manually for each function. There's an [issue](https://github.com/appwrite/appwrite/issues/3530) open at Appwrite to make this possible on a project level.
Navigate to the first function > Settings. Scroll down to Update Function Variables. Set the following vars:
* `APPWRITE_FUNCTION_ENDPOINT`: `http://172.17.0.1:8080/v1`
* `APPWRITE_DATABASE_ID`: This is the database ID of the database created above. Get it at Databases > click "Project ID" on the database created above.
* `BASE_URI`: This is the URL of the frontend
* `API_KEY`: The Appwrite API key you created above
You'll need to create these vars for each of the three functions.

View File

@ -45,8 +45,337 @@
],
"execute": [],
"events": [],
"schedule": "",
"schedule": "*/1 * * * *",
"timeout": 15
}
],
"collections": [
{
"$id": "servers",
"$createdAt": "2022-11-28T00:15:46.337+00:00",
"$updatedAt": "2022-11-28T00:16:55.140+00:00",
"$permissions": [
"read(\"users\")"
],
"databaseId": "637fd77aae471453b595",
"name": "servers",
"enabled": true,
"documentSecurity": false,
"attributes": [
{
"key": "server_url",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 1000,
"default": null
},
{
"key": "client_id",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 100,
"default": null
},
{
"key": "client_secret",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 100,
"default": null
},
{
"key": "app_access_token",
"type": "string",
"status": "available",
"required": false,
"array": false,
"size": 100,
"default": null
}
],
"indexes": [
{
"key": "server_url",
"type": "unique",
"status": "available",
"attributes": [
"server_url"
],
"orders": [
"DESC"
]
}
]
},
{
"$id": "hashtags",
"$createdAt": "2022-11-30T20:47:50.076+00:00",
"$updatedAt": "2022-11-30T20:47:50.076+00:00",
"$permissions": [],
"databaseId": "637fd77aae471453b595",
"name": "hashtags",
"enabled": true,
"documentSecurity": false,
"attributes": [
{
"key": "user",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 100,
"default": null
},
{
"key": "hashtag",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 500,
"default": null
},
{
"key": "lastseen",
"type": "datetime",
"status": "available",
"required": true,
"array": false,
"format": "",
"default": null
},
{
"key": "points",
"type": "integer",
"status": "available",
"required": true,
"array": false,
"min": 0,
"max": 9223372036854775807,
"default": null
}
],
"indexes": [
{
"key": "user_id",
"type": "key",
"status": "available",
"attributes": [
"user"
],
"orders": [
"DESC"
]
},
{
"key": "hashtag",
"type": "key",
"status": "available",
"attributes": [
"hashtag"
],
"orders": [
"DESC"
]
},
{
"key": "getUsersHashtags",
"type": "key",
"status": "available",
"attributes": [
"user",
"hashtag"
],
"orders": [
"DESC",
"DESC"
]
}
]
},
{
"$id": "index_history",
"$createdAt": "2022-11-30T21:05:31.728+00:00",
"$updatedAt": "2022-11-30T21:05:31.728+00:00",
"$permissions": [],
"databaseId": "637fd77aae471453b595",
"name": "index_history",
"enabled": true,
"documentSecurity": false,
"attributes": [
{
"key": "user_id",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 100,
"default": null
},
{
"key": "last_post_id",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 300,
"default": null
},
{
"key": "populating_new_index",
"type": "boolean",
"status": "available",
"required": false,
"array": false,
"default": true
},
{
"key": "first_post_id",
"type": "string",
"status": "available",
"required": false,
"array": false,
"size": 500,
"default": null
},
{
"key": "feed_name",
"type": "string",
"status": "available",
"required": true,
"array": false,
"elements": [
"home",
"favourites",
"bookmarks",
"local",
"federated",
"profile"
],
"format": "enum",
"default": null
}
],
"indexes": [
{
"key": "queryIndexHistory",
"type": "key",
"status": "available",
"attributes": [
"user_id",
"feed_name"
],
"orders": [
"DESC",
"DESC"
]
},
{
"key": "unique",
"type": "unique",
"status": "available",
"attributes": [
"user_id",
"feed_name"
],
"orders": [
"DESC",
"DESC"
]
}
]
},
{
"$id": "suggested",
"$createdAt": "2022-12-19T00:03:11.695+00:00",
"$updatedAt": "2022-12-19T23:18:53.402+00:00",
"$permissions": [],
"databaseId": "637fd77aae471453b595",
"name": "suggested",
"enabled": true,
"documentSecurity": true,
"attributes": [
{
"key": "post_link",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 2000,
"default": null
},
{
"key": "points",
"type": "integer",
"status": "available",
"required": true,
"array": false,
"min": 0,
"max": 9223372036854775807,
"default": null
},
{
"key": "user_id",
"type": "string",
"status": "available",
"required": true,
"array": false,
"size": 100,
"default": null
},
{
"key": "local_link",
"type": "string",
"status": "available",
"required": false,
"array": false,
"size": 2000,
"default": null
}
],
"indexes": [
{
"key": "point_sort",
"type": "key",
"status": "available",
"attributes": [
"points",
"user_id"
],
"orders": [
"DESC",
"DESC"
]
},
{
"key": "points",
"type": "key",
"status": "available",
"attributes": [
"points"
],
"orders": [
"DESC"
]
},
{
"key": "link_unique",
"type": "unique",
"status": "failed",
"attributes": [
"post_link"
],
"orders": [
"DESC"
]
}
]
}
]
}

View File

@ -28,8 +28,6 @@ require_once 'vendor/autoload.php';
If an error is thrown, a response with code 500 will be returned.
*/
// require_once 'threading/ThreadQueue.php';
include "crawl.php";
return function($req, $res) {

View File

@ -9,10 +9,95 @@ use Appwrite\Query;
class Suggestions {
public static function crawlFederatedFeed($user, $client) {
$request = new \GuzzleHttp\Client(['base_uri' => $user['prefs']['user_server_uri']]);
try {
$response = $request->request('GET', "/api/v1/timelines/public", [
'headers' => [
'Authorization' => 'Bearer ' . $user['prefs']['user_token']
]
]);
$responseBody = (string)$response->getBody();
$responseBody = json_decode($responseBody);
$statuses = [];
foreach($responseBody as $status) {
if(count($status->tags) == 0) {
continue;
}
$points = self::scoreStatus($status, $user, $client);
if($points['points'] !== 0) {
$statuses[] = $points;
}
}
usort($statuses, function ($a, $b) {
var_dump($a);
// return 0 <=> 0;
return $b['points'] <=> $a['points'];
});
$database = new Databases($client);
foreach($statuses as $status) {
$response = $request->request('GET', "/api/v2/search?type=statuses&resolve=true&q=" . $status['post'], [
'headers' => [
'Authorization' => 'Bearer ' . $user['prefs']['user_token']
]
]);
$responseBody = (string)$response->getBody();
$responseBody = json_decode($responseBody);
$result = $responseBody->statuses;
if(isset($result[0])) {
$result = $result[0];
var_dump($result);
$database->createDocument(getenv('APPWRITE_DATABASE_ID'), 'suggested', 'unique()', [
'post_link' => $status['post'],
'points' => $status['points'],
'local_link' => $user['prefs']['user_server_uri'] . "/@" . $result->account->acct . "/" . $result->id,
'user_id' => $user['$id']
], [
\Appwrite\Permission::read(\Appwrite\Role::user($user['$id']))
]);
}
}
return $statuses;
} catch (\GuzzleHttp\Exception\ClientException $e) {
return [$e->getMessage()];
}
}
public static function scoreStatus($status, $user_id, $client) {
public static function scoreStatus($status, $user, $client) {
$database = new Databases($client);
$query = new Query();
$points = 0;
foreach($status->tags as $tag) {
$record = $database->listDocuments(getenv('APPWRITE_DATABASE_ID'), 'hashtags', [
$query->equal('user', $user['$id']),
$query->equal('hashtag', $tag->name)
])['documents'];
if(isset($record[0])) {
$points += $record[0]['points'];
}
}
return [
'post' => $status->url,
'points' => $points
];
}
}