From 9f71e388ce8cf9c0b7d049d8b06643e0c485a574 Mon Sep 17 00:00:00 2001 From: Alex Hunt Date: Tue, 27 Nov 2018 22:02:08 -0800 Subject: [PATCH] Refactoring setup script and creating refresh API route --- TODO.txt | 20 ++++ application.py | 2 + blueprints/api.py | 10 +- setup.py | 223 ++++++++++++++++--------------------------- templates/joyce.html | 2 +- 5 files changed, 113 insertions(+), 144 deletions(-) diff --git a/TODO.txt b/TODO.txt index b924332..ce49c0b 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,3 +1,23 @@ +V1 REQUIREMENTS: + +- Finish Search + +- Add Image Support + + - Add Video Support + + - Add Audio Support + +- Deploy Process + +- Login System + +- Pagination + +- Add The Logo / Contributors + +- CSS Work + UX: - Disable Title Submit Button Without Title - Disable Modal Submit Button with input diff --git a/application.py b/application.py index 8c8f7a0..388742d 100644 --- a/application.py +++ b/application.py @@ -1,6 +1,7 @@ from flask import Flask import json import config +import setup from blueprints.joyce import joyce from blueprints.api import api @@ -12,6 +13,7 @@ params = { 'ENV': config.ENVIRONMENT, 'DEBUG': True, } + application.config.update(params) # Register blueprints diff --git a/blueprints/api.py b/blueprints/api.py index 08a4abb..248b7db 100644 --- a/blueprints/api.py +++ b/blueprints/api.py @@ -2,9 +2,9 @@ from flask import Blueprint, render_template, abort, jsonify, request from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth import json -import boto3 import sys import config +import setup sys.path.insert(0,'..') @@ -291,3 +291,11 @@ def search_text(): data = json.loads(request.data) results = es_search_text(data.get('data')) return jsonify(results) + +# +# Refresh ES +# TODO: Restrict to dev only +@api.route('/refresh/') +def refresh_es(): + setup.es_setup() + return 'Success!' diff --git a/setup.py b/setup.py index 5ba259c..423102b 100644 --- a/setup.py +++ b/setup.py @@ -2,10 +2,12 @@ from elasticsearch import Elasticsearch, RequestsHttpConnection from elasticsearch.helpers import bulk import config -if config.ENV_VARIABLE == 'local': +# Establish Elasticsearch Connection + +if config.ENVIRONMENT == 'local': es = Elasticsearch(config.ELASTICSEARCH_LOCAL_HOST) -if config.ENV_VARIABLE == 'staging': +if config.ENVIRONMENT == 'staging': es = Elasticsearch( hosts = config.ELASTICSEARCH_STAGING_HOST, http_auth = config.AWS_AUTH, @@ -14,19 +16,7 @@ if config.ENV_VARIABLE == 'staging': connection_class = RequestsHttpConnection ) -def get_chapter_text_from_seed_data(string): - string = string.encode('ascii', 'ignore') - fname = './seed_data/' + string + '.html' - HtmlFile = open(fname, 'r') - chapter_source = HtmlFile.read() - HtmlFile.close() - return chapter_source - -# DELETE INDEX: -es.indices.delete(index='chapters', ignore=[400, 404]) -es.indices.delete(index='notes', ignore=[400, 404]) -es.indices.delete(index='tags', ignore=[400, 404]) -print 'Elasticsearch index deleted!' +# Create Index Settings default_index_settings = { 'index': { @@ -80,138 +70,87 @@ chapter_index_settings = {'settings': default_index_settings, 'mappings': chapte note_index_settings = {'settings': default_index_settings, 'mappings': note_mappings} tag_index_settings = {'settings': default_index_settings, 'mappings': tag_mappings} -# TODO: Bulk create -es.indices.create(index='chapters', body=chapter_index_settings) -es.indices.create(index='notes', body=note_index_settings) -es.indices.create(index='tags', body=tag_index_settings) -print 'Elasticsearch index created!' +# Read Seed Data from Files + +def get_chapter_text_from_seed_data(string): + string = string.encode('utf-8', 'ignore') + fname = './seed_data/' + string + '.html' + HtmlFile = open(fname, 'r') + chapter_source = HtmlFile.read() + HtmlFile.close() + return chapter_source + +def build_es_chapter_op(id, number, title, chap_file): + return {'_op_type': 'index', '_id': id, '_source': { + 'number': number, + 'title': title, + 'html_source': get_chapter_text_from_seed_data(chap_file) + } + } + +def build_es_note_op(id, title, text): + return {'_op_type': 'index', '_id': id, '_source': { + 'title': title, + 'html_source': text, + } + } -# Sample data SAMPLE_CHAPTERS = [ - {'_op_type': 'index', '_id': 'AWNM3N3mxgFi4og697un', '_source': { - 'number': 1, - 'title': 'Telemachus', - 'html_source': get_chapter_text_from_seed_data('telem') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vA', '_source': { - 'number': 2, - 'title': 'Nestor', - 'html_source': get_chapter_text_from_seed_data('nestor') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vB', '_source': { - 'number': 3, - 'title': 'Proteus', - 'html_source': get_chapter_text_from_seed_data('proteus') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vC', '_source': { - 'number': 4, - 'title': 'Calyspo', - 'html_source': get_chapter_text_from_seed_data('calypso') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vD', '_source': { - 'number': 5, - 'title': 'Lotus Eaters', - 'html_source': get_chapter_text_from_seed_data('lotus') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vE', '_source': { - 'number': 6, - 'title': 'Hades', - 'html_source': get_chapter_text_from_seed_data('hades') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vF', '_source': { - 'number': 7, - 'title': 'Aeolus', - 'html_source': get_chapter_text_from_seed_data('aeolus') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vG', '_source': { - 'number': 8, - 'title': 'Lestrygonians', - 'html_source': get_chapter_text_from_seed_data('lestry') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vH', '_source': { - 'number': 9, - 'title': 'Scylla and Charybdis', - 'html_source': get_chapter_text_from_seed_data('scylla') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vI', '_source': { - 'number': 10, - 'title': 'Wandering Rocks', - 'html_source': get_chapter_text_from_seed_data('wrocks') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vJ', '_source': { - 'number': 11, - 'title': 'Sirens', - 'html_source': get_chapter_text_from_seed_data('sirens') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vK', '_source': { - 'number': 12, - 'title': 'Cyclops', - 'html_source': get_chapter_text_from_seed_data('cyclops') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vL', '_source': { - 'number': 13, - 'title': 'Nausicaa', - 'html_source': get_chapter_text_from_seed_data('nausicaa') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vM', '_source': { - 'number': 14, - 'title': 'Oxen of the Sun', - 'html_source': get_chapter_text_from_seed_data('oxen') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vN', '_source': { - 'number': 15, - 'title': 'Circe', - 'html_source': get_chapter_text_from_seed_data('circe') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vO', '_source': { - 'number': 16, - 'title': 'Eumaeus', - 'html_source': get_chapter_text_from_seed_data('eumaeus') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vP', '_source': { - 'number': 17, - 'title': 'Ithaca', - 'html_source': get_chapter_text_from_seed_data('ithaca') - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vQ', '_source': { - 'number': 18, - 'title': 'Penelope', - 'html_source': get_chapter_text_from_seed_data('penelope') - }, - }, + build_es_chapter_op('AWNM3N3mxgFi4og697un', 1, 'Telemachus', 'telem'), + build_es_chapter_op('AWNmqpdHxgFi4og697vA', 2, 'Nestor', 'nestor'), + build_es_chapter_op('AWNmqpdHxgFi4og697vB', 3, 'Proteus', 'proteus'), + build_es_chapter_op('AWNmqpdHxgFi4og697vC', 4, 'Calyspo', 'calypso'), + build_es_chapter_op('AWNmqpdHxgFi4og697vD', 5, 'Lotus Eaters', 'lotus'), + build_es_chapter_op('AWNmqpdHxgFi4og697vE', 6, 'Hades', 'hades'), + build_es_chapter_op('AWNmqpdHxgFi4og697vF', 7, 'Aeolus', 'aeolus'), + build_es_chapter_op('AWNmqpdHxgFi4og697vG', 8, 'Lestrygonians', 'lestry'), + build_es_chapter_op('AWNmqpdHxgFi4og697vH', 9, 'Scylla and Charybdis', 'scylla'), + build_es_chapter_op('AWNmqpdHxgFi4og697vI', 10, 'Wandering Rocks', 'wrocks'), + build_es_chapter_op('AWNmqpdHxgFi4og697vJ', 11, 'Sirens', 'sirens'), + build_es_chapter_op('AWNmqpdHxgFi4og697vK', 12, 'Cyclops', 'cyclops'), + build_es_chapter_op('AWNmqpdHxgFi4og697vL', 13, 'Nausicaa', 'nausicaa'), + build_es_chapter_op('AWNmqpdHxgFi4og697vM', 14, 'Oxen of the Sun', 'oxen'), + build_es_chapter_op('AWNmqpdHxgFi4og697vN', 15, 'Circe', 'circe'), + build_es_chapter_op('AWNmqpdHxgFi4og697vO', 16, 'Eumaeus', 'eumaeus'), + build_es_chapter_op('AWNmqpdHxgFi4og697vP', 17, 'Ithaca', 'ithaca'), + build_es_chapter_op('AWNmqpdHxgFi4og697vQ', 18, 'Penelope', 'penelope'), ] SAMPLE_NOTES = [ - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vR', '_source': { - 'title': 'Kinch', - 'html_source': 'A knife' - }, - }, - {'_op_type': 'index', '_id': 'AWNmqpdHxgFi4og697vS', '_source': { - 'title': 'Lighthouse', - 'html_source': 'A lighthouse' - }, - }, + build_es_note_op('AWNmqpdHxgFi4og697vR', 'Kinch', 'A knife'), + build_es_note_op('AWNmqpdHxgFi4og697vS', 'Lighthouse', 'A lighthouse'), ] -bulk(es, SAMPLE_CHAPTERS, index='chapters', doc_type='doc') -bulk(es, SAMPLE_NOTES, index='notes', doc_type='doc') -print('Successfully loaded sample data!') +# Manipulate Indices + +def delete_index(index): + es.indices.delete(index=index, ignore=[400, 404]) + +def create_index(index, settings): + es.indices.create(index=index, body=settings) + +def index_seed_docs(index, docs): + bulk(es, docs, index=index, doc_type='doc') + +def refresh_all_indices(): + delete_index('chapters') + delete_index('notes') + delete_index('tags') + print 'Elasticsearch index deleted!' + create_index('chapters', chapter_index_settings) + create_index('notes', note_index_settings) + create_index('tags', tag_index_settings) + print 'Elasticsearch index created!' + +def refresh_seed_data(): + index_seed_docs('chapters', SAMPLE_CHAPTERS) + index_seed_docs('notes', SAMPLE_NOTES) + print 'Successfully loaded sample data!' + +def es_setup(): + refresh_all_indices() + refresh_seed_data() + +if __name__ == "__main__": + es_setup() diff --git a/templates/joyce.html b/templates/joyce.html index 512d36f..61b8aa1 100644 --- a/templates/joyce.html +++ b/templates/joyce.html @@ -6,7 +6,7 @@ - +