simple news crawler from fb

This commit is contained in:
deflax 2017-05-25 08:54:33 +03:00
parent 17179e8deb
commit 810ab906c4
18 changed files with 217 additions and 142 deletions

View file

@ -1,4 +1,4 @@
Copyright (c) 2015-2016 deflax Copyright (c) 2015-2017 deflax.net
This software is provided 'as-is', without any express or implied This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages warranty. In no event will the authors be held liable for any damages

View file

@ -34,6 +34,9 @@ babel.init_app(app)
from .main import main as main_blueprint from .main import main as main_blueprint
app.register_blueprint(main_blueprint) app.register_blueprint(main_blueprint)
from .news import news as news_blueprint
app.register_blueprint(news_blueprint, url_prefix='/news')
from .auth import auth as auth_blueprint from .auth import auth as auth_blueprint
app.register_blueprint(auth_blueprint, url_prefix='/auth') app.register_blueprint(auth_blueprint, url_prefix='/auth')
@ -60,7 +63,7 @@ class CustomJSONEncoder(JSONEncoder):
app.json_encoder = CustomJSONEncoder app.json_encoder = CustomJSONEncoder
#if app.debug: #TODO: if app.debug:
if not app.debug: if not app.debug:
import logging import logging
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
@ -69,7 +72,7 @@ if not app.debug:
file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]')) file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
app.logger.addHandler(file_handler) app.logger.addHandler(file_handler)
app.logger.setLevel(logging.DEBUG) app.logger.setLevel(logging.DEBUG)
app.logger.info('Proxadmin started.') #app.logger.info('Proxadmin started.')
@app.errorhandler(403) @app.errorhandler(403)
def forbidden(e): def forbidden(e):

3
app/news/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from flask import Blueprint
news = Blueprint('news', __name__)
from . import routes

40
app/news/routes.py Normal file
View file

@ -0,0 +1,40 @@
from flask import render_template, abort, redirect, url_for, abort, flash, request, current_app, make_response, g
from . import news
from functools import lru_cache
import requests
import json
from facepy import GraphAPI
@lru_cache(maxsize=8)
def get_fb_token():
payload = {'grant_type': 'client_credentials', 'client_id': current_app.config['FB_APP_ID'], 'client_secret': current_app.config['FB_APP_SECRET']}
response = requests.post('https://graph.facebook.com/oauth/access_token?', params = payload)
fbjson = json.loads(response.text)
fbtoken = fbjson['access_token']
current_app.logger.info("FB Token: {}".format(fbtoken))
return fbtoken
@news.route("/all", methods=['GET'])
def all():
graph = GraphAPI(get_fb_token())
#current_app.logger.info(get_fb_token.cache_info())
page_id = current_app.config['FB_PAGE_ID']
datas = graph.get(page_id+'/posts?fields=id,message,created_time,link,icon', page=True, retry=3)
posts = []
for data in datas:
posts.append(data)
content=posts[0]['data']
return render_template('news/facebook.html', result=content)
@news.route('/latest')
def latest():
graph = GraphAPI(get_fb_token())
page_id = current_app.config['FB_PAGE_ID']
latest = graph.get(page_id+'/posts?limit=1&fields=id,message,created_time,link', page=True, retry=3)
posts = []
for data in latest:
posts.append(data)
content=posts[0]['data']
return render_template('news/latest.html', result=content)

View file

@ -51,7 +51,7 @@ def profile():
form.twofactor.data = current_user.twofactor form.twofactor.data = current_user.twofactor
wallet = "%.2f" % round(current_user.wallet, 3) wallet = "%.2f" % round(current_user.wallet, 3)
current_app.logger.info('wallet: ' + wallet) current_app.logger.info('[{}] wallet: {}'.format(current_user.email, wallet))
return render_template('settings/profile.html', page=page, form=form, wallet=wallet) return render_template('settings/profile.html', page=page, form=form, wallet=wallet)

View file

@ -1,12 +0,0 @@
{% macro render_field(field) %}
<dt>{{ field.label }}
<dd>{{ field(**kwargs)|safe }}
{% if field.errors %}
<ul class=errors>
{% for error in field.errors %}
<li>{{ error }}</li>
{% endfor %}
</ul>
{% endif %}
</dd>
{% endmacro %}

View file

@ -1,57 +0,0 @@
{% extends "base.html" %}
{% block page_content %}
<h1>Index</h1>
<div id="myCarousel" class="carousel slide" data-ride="carousel">
<!-- Indicators -->
<ol class="carousel-indicators">
<li data-target="#myCarousel" data-slide-to="0" class="active"></li>
<li data-target="#myCarousel" data-slide-to="1"></li>
<li data-target="#myCarousel" data-slide-to="2"></li>
<li data-target="#myCarousel" data-slide-to="3"></li>
</ol>
<!-- Wrapper for slides -->
<div class="carousel-inner" role="listbox">
<div class="item active">
<img src="static/images/1.jpg" alt="Chania">
</div>
<div class="item">
<img src="static/images/2.jpg" alt="Chania">
</div>
<div class="item">
<img src="static/images/3.jpg" alt="Flower">
</div>
<div class="item">
<img src="static/images/4.jpg" alt="Flower">
</div>
</div>
<!-- Left and right controls -->
<a class="left carousel-control" href="#myCarousel" role="button" data-slide="prev">
<span class="glyphicon glyphicon-chevron-left" aria-hidden="true"></span>
<span class="sr-only">Previous</span>
</a>
<a class="right carousel-control" href="#myCarousel" role="button" data-slide="next">
<span class="glyphicon glyphicon-chevron-right" aria-hidden="true"></span>
<span class="sr-only">Next</span>
</a>
</div>
<div class="row">
<div class="col-md-4">
<p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
</div>
<div class="col-md-4">
<p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
</div>
<div class="col-md-4">
<p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
</div>
</div>
{% endblock %}

View file

@ -6,8 +6,6 @@
{% block title %}Cloud Builder - Datapoint.bg{% endblock %} {% block title %}Cloud Builder - Datapoint.bg{% endblock %}
{% block html_attribs %} lang="en"{% endblock %}
{% block head %} {% block head %}
{{ super() }} {{ super() }}
<link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}" type="image/x-icon"> <link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}" type="image/x-icon">

View file

@ -7,19 +7,13 @@
<p>tel: +359 (0) 32 398 295 <br /> email: <a href="mailto:office@datapoint.bg">office@datapoint.bg</a></p> <p>tel: +359 (0) 32 398 295 <br /> email: <a href="mailto:office@datapoint.bg">office@datapoint.bg</a></p>
</div><!--/one four--> </div><!--/one four-->
<div class="one_four">
<h4>Latest Tweet</h4>
<p><a href="#">@datapoint</a> Продавам пирони. Изгодно.</p>
<span class="tweet_day">2 Days ago</span>
</div><!--/one four-->
<div class="one_four"> <div class="one_four">
<h4>Datapoint</h4> <h4>Datapoint</h4>
<p>We are a dedicated team working to bring you the best hosting in the business. <p>We are a dedicated team working to bring you the best hosting in the business.
We are passionate about what we do, and why we do it.<br> <a href="#">Read More…</a></p> We are passionate about what we do, and why we do it.<br> <a href="#">Read More…</a></p>
</div><!--/one four--> </div><!--/one four-->
<div class="one_four last_col"> <div class="one_four">
<h4>Newsletter Signup</h4> <h4>Newsletter Signup</h4>
<p>Join our newsletter to stay informed on our latest updates and sales.</p> <p>Join our newsletter to stay informed on our latest updates and sales.</p>
<form action="#" method="post"> <form action="#" method="post">
@ -30,5 +24,10 @@
</form> </form>
</div><!--/one four--> </div><!--/one four-->
<div class="one_four last_col" id="newsmenu">
<!--#include virtual="https://www.datapoint.bg/news/latest" -->
</div><!--/one four-->
</div><!--/page wrap--> </div><!--/page wrap-->
</div><!--/footer cols--> </div><!--/footer cols-->

View file

@ -42,8 +42,15 @@
}; };
makeBSS('.bss-slides', opts); makeBSS('.bss-slides', opts);
</script> </script>
<script type="text/javascript">
$(document).ready(function() {
$('#newsmenu').load('https://www.datapoint.bg/news/latest');
});
</script>
{% endblock %} {% endblock %}
{% block page_content %} {% block page_content %}
<!-- <!--
<div class="row"> <div class="row">

View file

@ -0,0 +1,14 @@
{% extends "base.html" %}
{% block page_content %}
<h1>News</h1>
<div class="row">
<ul id="navigation">
{% for item in result %}
<li><a target="_blank" href="{{ item['link'] }}">{{ item['message'] }}</a> {{ item['created_time'] }}<br />
</li><br />
{% endfor %}
</ul>
</div>
{% endblock %}

View file

@ -0,0 +1,8 @@
{% block page_content %}
<h4>News</h4>
{% for item in result %}
<p><a target="_blank" href="{{ item['link'] }}">{{ item['message'] }}</a></p>
<span class="tweet_day">{{ item['created_time'] }}</span>
{% endfor %}<br />
<a href="/news/all">>>></a>
{% endblock %}

View file

@ -43,40 +43,40 @@ $("#{{ form.org_account.id }}").click(function() {
<div class="panel-body"> <div class="panel-body">
<form method="POST" action="{{ url_for('settings.profile') }}"> <form method="POST" action="{{ url_for('settings.profile') }}">
<p> <p>
{{ form.name.label }}<br />{{ form.name }}<br /> {{ form.name.label }}<br />{{ form.name(style="width: 80%;") }}<br />
{% for error in form.name.errors %} {% for error in form.name.errors %}
{{ error }}<br /> {{ error }}<br />
{% endfor %} {% endfor %}
</p> </p>
<p> <p>
{{ form.address.label }}<br /> {{ form.address }}<br /> {{ form.address.label }}<br /> {{ form.address(style="width: 80%;") }}<br />
{% for error in form.address.errors %} {% for error in form.address.errors %}
{{ error }}<br /> {{ error }}<br />
{% endfor %} {% endfor %}
</p> </p>
<p> <p>
{{ form.city.label }}<br />{{ form.city }}<br /> {{ form.city.label }}<br />{{ form.city(style="width: 80%;") }}<br />
{% for error in form.city.errors %} {% for error in form.city.errors %}
{{ error }}<br /> {{ error }}<br />
{% endfor %} {% endfor %}
</p> </p>
<p> <p>
{{ form.country.label }}<br /> {{ form.country(style="width: 80%;") }}<br />
{% for error in form.country.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
{{ form.postcode.label }}<br />{{ form.postcode }}<br /> {{ form.postcode.label }}<br />{{ form.postcode }}<br />
{% for error in form.postcode.errors %} {% for error in form.postcode.errors %}
{{ error }}<br /> {{ error }}<br />
{% endfor %} {% endfor %}
</p> </p>
<p>
{{ form.country.label }}<br /> {{ form.country }}<br />
{% for error in form.country.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p> <p>
{{ form.phone.label }}<br /> {{ form.phone }}<br /> {{ form.phone.label }}<br /> {{ form.phone }}<br />
{% for error in form.phone.errors %} {% for error in form.phone.errors %}
@ -96,19 +96,19 @@ $("#{{ form.org_account.id }}").click(function() {
<div class="panel-heading">Данни за юридическо лице</div> <div class="panel-heading">Данни за юридическо лице</div>
<div class="panel-body"> <div class="panel-body">
<p> <p>
{{ form.org_companyname.label }}<br />{{ form.org_companyname }}<br /> {{ form.org_companyname.label }}<br />{{ form.org_companyname(style="width: 80%;") }}<br />
{% for error in form.org_companyname.errors %} {% for error in form.org_companyname.errors %}
{{ error }}<br /> {{ error }}<br />
{% endfor %} {% endfor %}
</p> </p>
<p> <p>
{{ form.org_regaddress.label }}<br />{{ form.org_regaddress }}<br /> {{ form.org_regaddress.label }}<br />{{ form.org_regaddress(style="width: 80%;") }}<br />
{% for error in form.org_regaddress.errors %} {% for error in form.org_regaddress.errors %}
{{ error }}<br /> {{ error }}<br />
{% endfor %} {% endfor %}
</p> </p>
<p> <p>
{{ form.org_responsible.label }}<br />{{ form.org_responsible }}<br /> {{ form.org_responsible.label }}<br />{{ form.org_responsible(style="width: 80%;") }}<br />
{% for error in form.org_responsible.errors %} {% for error in form.org_responsible.errors %}
{{ error }}<br /> {{ error }}<br />
{% endfor %} {% endfor %}

View file

@ -1,5 +1,16 @@
{% extends "base.html" %} {% extends "base.html" %}
{% block scripts %}
{{ super() }}
<script type="text/javascript">
//$(function() {
// $('.confirm').click(function() {
// return window.confirm("Are you sure?");
// });
//});
</script>
{% endblock %}
{% block page_content %} {% block page_content %}
<script> <script>
@ -14,8 +25,9 @@ addEventListener("DOMContentLoaded", function() {
button.addEventListener("click", function(e) { button.addEventListener("click", function(e) {
// When a click happens, stop the button // When a click happens, stop the button
// from submitting our form (if we have one) // from submitting our form (if we have one)
e.preventDefault(); e.preventDefault();
if (window.confirm("Are you sure?")) {
var clickedButton = e.target; var clickedButton = e.target;
var command = clickedButton.value; var command = clickedButton.value;
var vmid = clickedButton.getAttribute('vmid'); var vmid = clickedButton.getAttribute('vmid');
@ -35,13 +47,13 @@ addEventListener("DOMContentLoaded", function() {
request.open("GET", "/vmanager/" + command + "/" + vmid, true); request.open("GET", "/vmanager/" + command + "/" + vmid, true);
// and then we send it off // and then we send it off
request.send(); request.send();
}); //alert("command " + command + " executed.");
} }
});
}
}, true); }, true);
</script> </script>
<style type="text/css"> <style type="text/css">
.tg {border-collapse:collapse;border-spacing:0;} .tg {border-collapse:collapse;border-spacing:0;}
.tg td{font-family:Arial, sans-serif;font-size:14px;padding:1px 1px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;} .tg td{font-family:Arial, sans-serif;font-size:14px;padding:1px 1px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;}
@ -109,8 +121,8 @@ addEventListener("DOMContentLoaded", function() {
<br /> <br />
{% if status[deploy.machine_id] == 'running' %} {% if status[deploy.machine_id] == 'running' %}
<button class="btn btn-default btn-info" onclick="window.open('/vmanager/vmvnc/{{ deploy.machine_id }}', '_blank');"><span class="glyphicon glyphicon-console" aria-hidden="true"></span> Console</button> <button class="btn btn-default btn-info" onclick="window.open('/vmanager/vmvnc/{{ deploy.machine_id }}', '_blank');"><span class="glyphicon glyphicon-console" aria-hidden="true"></span> Console</button>
<button class="command command-vmshutdown btn btn-default btn-warning" value="vmshutdown" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-off" aria-hidden="true"></span> Shutdown</button> <button class="confirm command command-vmshutdown btn btn-default btn-warning" value="vmshutdown" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-off" aria-hidden="true"></span> Shutdown</button>
<button class="command command-vmstop btn btn-default btn-danger" value="vmstop" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-alert" aria-hidden="true"></span> Force Stop</button> <button class="confirm command command-vmstop btn btn-default btn-danger" value="vmstop" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-alert" aria-hidden="true"></span> Force Stop</button>
{% else %} {% else %}
<button class="command command-vmstart btn btn-default btn-success" value="vmstart" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-play" aria-hidden="true"></span> Start</button> <button class="command command-vmstart btn btn-default btn-success" value="vmstart" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-play" aria-hidden="true"></span> Start</button>
{% endif %} {% endif %}

View file

@ -58,7 +58,7 @@ def deploy(product_id=None):
# return redirect(url_for('uinvoice.addfunds')) # return redirect(url_for('uinvoice.addfunds'))
if current_user.name is None: if current_user.name is None:
flash('Моля обновете информацията за профила') flash('Please update profile info for this operation.')
return redirect(url_for('uinvoice.profile')) return redirect(url_for('uinvoice.profile'))
page = { 'title': 'Deploy' } page = { 'title': 'Deploy' }
@ -123,17 +123,29 @@ def deploy(product_id=None):
@login_required @login_required
def dashboard(): def dashboard():
deployments = current_user.inv_deployments.order_by(Deployment.date_created.desc()).all() deployments = current_user.inv_deployments.order_by(Deployment.date_created.desc()).all()
inv_deploycubeids = []
inv_deployments = [] inv_deploynames = []
for invcls in deployments: for invcls in deployments:
if invcls.enabled == True: if invcls.enabled == True:
inv_deployments.extend([invcls.machine_id]) inv_deploycubeids.extend([invcls.machine_id])
#if not inv_deployments: inv_deploynames.extend([invcls.machine_alias])
# return render_template('vmanager/empty_dashboard.html')
contracts = current_user.inv_contracts.order_by(Contract.date_created.desc()).all()
inv_contracts = []
for invcls in contracts:
if invcls.enabled == True:
inv_contracts.extend([invcls.template])
domains = current_user.inv_domains.order_by(Domain.date_created.desc()).all()
inv_domains = []
for invcls in domains:
if invcls.enabled == True:
inv_domains.extend([invcls.fqdn])
#extract rrd and status from the deployments
rrd = {} rrd = {}
statuses = {} statuses = {}
for cubeid in inv_deployments: for cubeid in inv_deploycubeids:
rrd[cubeid] = {} rrd[cubeid] = {}
try: try:
query = contact_proxmaster({}, 'vmrrd', cubeid) query = contact_proxmaster({}, 'vmrrd', cubeid)
@ -155,20 +167,7 @@ def dashboard():
send_email(current_app.config['MAIL_USERNAME'], 'Cube {} is unreachable'.format(cubeid), send_email(current_app.config['MAIL_USERNAME'], 'Cube {} is unreachable'.format(cubeid),
'vmanager/email/adm_unreachable', user=current_user, cubeid=cubeid ) 'vmanager/email/adm_unreachable', user=current_user, cubeid=cubeid )
contracts = current_user.inv_contracts.order_by(Contract.date_created.desc()).all() current_app.logger.info('[{}] deployments: {}, domains: {}, services: {}'.format(current_user.email, inv_deploynames, inv_contracts, inv_domains ))
#inv_contracts = []
#for invcls in contracts:
# if invcls.enabled == True:
# inv_contracts.extend([invcls.template])
domains = current_user.inv_domains.order_by(Domain.date_created.desc()).all()
#inv_domains = []
#for invcls in domains:
# if invcls.enabled == True:
# inv_domains.extend([invcls.fqdn])
#print(statuses)
return render_template('vmanager/dashboard.html', rrd=rrd, status=statuses, inv_deployments=deployments, inv_contracts=contracts, inv_domains=domains) return render_template('vmanager/dashboard.html', rrd=rrd, status=statuses, inv_deployments=deployments, inv_contracts=contracts, inv_domains=domains)
@vmanager.route('/vmvnc/<int:vmid>') @vmanager.route('/vmvnc/<int:vmid>')

3
babel.cfg Normal file
View file

@ -0,0 +1,3 @@
[python: **.py]
[jinja2: **/templates/**.html]
extensions=jinja2.ext.autoescape,jinja2.ext.with_

36
before_upgrade.txt Normal file
View file

@ -0,0 +1,36 @@
alembic==0.9.0
Babel==2.3.4
blinker==1.4
click==6.7
dnspython==1.15.0
dnspython3==1.15.0
dominate==2.3.1
Flask==0.12
Flask-Babel==0.11.1
Flask-Bootstrap==3.3.7.1
Flask-Login==0.4.0
Flask-Mail==0.9.1
Flask-Migrate==2.0.3
Flask-Script==2.0.5
Flask-SQLAlchemy==2.2
Flask-WTF==0.14.2
gunicorn==19.7.0
iso3166==0.8
itsdangerous==0.24
Jinja2==2.9.5
Mako==1.0.6
MarkupSafe==0.23
onetimepass==1.0.1
pkg-resources==0.0.0
psycopg2==2.6.2
PyQRCode==1.2.1
python-editor==1.0.3
pytz==2016.10
requests==2.13.0
schedule==0.4.2
six==1.10.0
sortedcontainers==1.5.7
SQLAlchemy==1.1.6
visitor==0.1.3
Werkzeug==0.11.15
WTForms==2.1

View file

@ -1,17 +1,39 @@
Flask alembic==0.9.2
Flask-Login Babel==2.4.0
Flask-Bootstrap blinker==1.4
Flask-WTF click==6.7
Flask-Script dnspython==1.15.0
Flask-Migrate dnspython3==1.15.0
Flask-Mail dominate==2.3.1
Flask-SQLAlchemy Flask==0.12.2
Flask-Babel Flask-Babel==0.11.2
psycopg2 Flask-Bootstrap==3.3.7.1
itsdangerous Flask-Login==0.4.0
requests Flask-Mail==0.9.1
sortedcontainers Flask-Migrate==2.0.3
iso3166 Flask-Script==2.0.5
pyqrcode Flask-SQLAlchemy==2.2
onetimepass Flask-WTF==0.14.2
schedule gunicorn==19.7.1
iso3166==0.8
itsdangerous==0.24
Jinja2==2.9.6
Mako==1.0.6
MarkupSafe==1.0
onetimepass==1.0.1
pkg-resources==0.0.0
psycopg2==2.7.1
PyQRCode==1.2.1
python-dateutil==2.6.0
python-editor==1.0.3
pytz==2017.2
requests==2.14.2
schedule==0.4.2
six==1.10.0
sortedcontainers==1.5.7
SQLAlchemy==1.1.10
visitor==0.1.3
Werkzeug==0.12.2
WTForms==2.1
facepy==1.0.8
appdirs==1.4.3