simple news crawler from fb

This commit is contained in:
deflax 2017-05-25 08:54:33 +03:00
parent 17179e8deb
commit 810ab906c4
18 changed files with 217 additions and 142 deletions

View file

@ -1,4 +1,4 @@
Copyright (c) 2015-2016 deflax
Copyright (c) 2015-2017 deflax.net
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages

View file

@ -34,6 +34,9 @@ babel.init_app(app)
from .main import main as main_blueprint
app.register_blueprint(main_blueprint)
from .news import news as news_blueprint
app.register_blueprint(news_blueprint, url_prefix='/news')
from .auth import auth as auth_blueprint
app.register_blueprint(auth_blueprint, url_prefix='/auth')
@ -60,7 +63,7 @@ class CustomJSONEncoder(JSONEncoder):
app.json_encoder = CustomJSONEncoder
#if app.debug:
#TODO: if app.debug:
if not app.debug:
import logging
from logging.handlers import RotatingFileHandler
@ -69,7 +72,7 @@ if not app.debug:
file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.DEBUG)
app.logger.info('Proxadmin started.')
#app.logger.info('Proxadmin started.')
@app.errorhandler(403)
def forbidden(e):

3
app/news/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from flask import Blueprint
news = Blueprint('news', __name__)
from . import routes

40
app/news/routes.py Normal file
View file

@ -0,0 +1,40 @@
from flask import render_template, abort, redirect, url_for, abort, flash, request, current_app, make_response, g
from . import news
from functools import lru_cache
import requests
import json
from facepy import GraphAPI
@lru_cache(maxsize=8)
def get_fb_token():
payload = {'grant_type': 'client_credentials', 'client_id': current_app.config['FB_APP_ID'], 'client_secret': current_app.config['FB_APP_SECRET']}
response = requests.post('https://graph.facebook.com/oauth/access_token?', params = payload)
fbjson = json.loads(response.text)
fbtoken = fbjson['access_token']
current_app.logger.info("FB Token: {}".format(fbtoken))
return fbtoken
@news.route("/all", methods=['GET'])
def all():
graph = GraphAPI(get_fb_token())
#current_app.logger.info(get_fb_token.cache_info())
page_id = current_app.config['FB_PAGE_ID']
datas = graph.get(page_id+'/posts?fields=id,message,created_time,link,icon', page=True, retry=3)
posts = []
for data in datas:
posts.append(data)
content=posts[0]['data']
return render_template('news/facebook.html', result=content)
@news.route('/latest')
def latest():
graph = GraphAPI(get_fb_token())
page_id = current_app.config['FB_PAGE_ID']
latest = graph.get(page_id+'/posts?limit=1&fields=id,message,created_time,link', page=True, retry=3)
posts = []
for data in latest:
posts.append(data)
content=posts[0]['data']
return render_template('news/latest.html', result=content)

View file

@ -51,7 +51,7 @@ def profile():
form.twofactor.data = current_user.twofactor
wallet = "%.2f" % round(current_user.wallet, 3)
current_app.logger.info('wallet: ' + wallet)
current_app.logger.info('[{}] wallet: {}'.format(current_user.email, wallet))
return render_template('settings/profile.html', page=page, form=form, wallet=wallet)

View file

@ -1,12 +0,0 @@
{% macro render_field(field) %}
<dt>{{ field.label }}
<dd>{{ field(**kwargs)|safe }}
{% if field.errors %}
<ul class=errors>
{% for error in field.errors %}
<li>{{ error }}</li>
{% endfor %}
</ul>
{% endif %}
</dd>
{% endmacro %}

View file

@ -1,57 +0,0 @@
{% extends "base.html" %}
{% block page_content %}
<h1>Index</h1>
<div id="myCarousel" class="carousel slide" data-ride="carousel">
<!-- Indicators -->
<ol class="carousel-indicators">
<li data-target="#myCarousel" data-slide-to="0" class="active"></li>
<li data-target="#myCarousel" data-slide-to="1"></li>
<li data-target="#myCarousel" data-slide-to="2"></li>
<li data-target="#myCarousel" data-slide-to="3"></li>
</ol>
<!-- Wrapper for slides -->
<div class="carousel-inner" role="listbox">
<div class="item active">
<img src="static/images/1.jpg" alt="Chania">
</div>
<div class="item">
<img src="static/images/2.jpg" alt="Chania">
</div>
<div class="item">
<img src="static/images/3.jpg" alt="Flower">
</div>
<div class="item">
<img src="static/images/4.jpg" alt="Flower">
</div>
</div>
<!-- Left and right controls -->
<a class="left carousel-control" href="#myCarousel" role="button" data-slide="prev">
<span class="glyphicon glyphicon-chevron-left" aria-hidden="true"></span>
<span class="sr-only">Previous</span>
</a>
<a class="right carousel-control" href="#myCarousel" role="button" data-slide="next">
<span class="glyphicon glyphicon-chevron-right" aria-hidden="true"></span>
<span class="sr-only">Next</span>
</a>
</div>
<div class="row">
<div class="col-md-4">
<p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
</div>
<div class="col-md-4">
<p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
</div>
<div class="col-md-4">
<p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
</div>
</div>
{% endblock %}

View file

@ -6,8 +6,6 @@
{% block title %}Cloud Builder - Datapoint.bg{% endblock %}
{% block html_attribs %} lang="en"{% endblock %}
{% block head %}
{{ super() }}
<link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}" type="image/x-icon">

View file

@ -7,19 +7,13 @@
<p>tel: +359 (0) 32 398 295 <br /> email: <a href="mailto:office@datapoint.bg">office@datapoint.bg</a></p>
</div><!--/one four-->
<div class="one_four">
<h4>Latest Tweet</h4>
<p><a href="#">@datapoint</a> Продавам пирони. Изгодно.</p>
<span class="tweet_day">2 Days ago</span>
</div><!--/one four-->
<div class="one_four">
<h4>Datapoint</h4>
<p>We are a dedicated team working to bring you the best hosting in the business.
We are passionate about what we do, and why we do it.<br> <a href="#">Read More…</a></p>
</div><!--/one four-->
<div class="one_four last_col">
<div class="one_four">
<h4>Newsletter Signup</h4>
<p>Join our newsletter to stay informed on our latest updates and sales.</p>
<form action="#" method="post">
@ -29,6 +23,11 @@
</div><!--/form field-->
</form>
</div><!--/one four-->
<div class="one_four last_col" id="newsmenu">
<!--#include virtual="https://www.datapoint.bg/news/latest" -->
</div><!--/one four-->
</div><!--/page wrap-->
</div><!--/footer cols-->

View file

@ -42,8 +42,15 @@
};
makeBSS('.bss-slides', opts);
</script>
<script type="text/javascript">
$(document).ready(function() {
$('#newsmenu').load('https://www.datapoint.bg/news/latest');
});
</script>
{% endblock %}
{% block page_content %}
<!--
<div class="row">

View file

@ -0,0 +1,14 @@
{% extends "base.html" %}
{% block page_content %}
<h1>News</h1>
<div class="row">
<ul id="navigation">
{% for item in result %}
<li><a target="_blank" href="{{ item['link'] }}">{{ item['message'] }}</a> {{ item['created_time'] }}<br />
</li><br />
{% endfor %}
</ul>
</div>
{% endblock %}

View file

@ -0,0 +1,8 @@
{% block page_content %}
<h4>News</h4>
{% for item in result %}
<p><a target="_blank" href="{{ item['link'] }}">{{ item['message'] }}</a></p>
<span class="tweet_day">{{ item['created_time'] }}</span>
{% endfor %}<br />
<a href="/news/all">>>></a>
{% endblock %}

View file

@ -41,42 +41,42 @@ $("#{{ form.org_account.id }}").click(function() {
<div class="panel panel-default">
<div class="panel-heading">Данни на профила</div>
<div class="panel-body">
<form method="POST" action="{{ url_for('settings.profile') }}">
<form method="POST" action="{{ url_for('settings.profile') }}">
<p>
{{ form.name.label }}<br />{{ form.name }}<br />
{{ form.name.label }}<br />{{ form.name(style="width: 80%;") }}<br />
{% for error in form.name.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
{{ form.address.label }}<br /> {{ form.address }}<br />
{{ form.address.label }}<br /> {{ form.address(style="width: 80%;") }}<br />
{% for error in form.address.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
{{ form.city.label }}<br />{{ form.city }}<br />
{{ form.city.label }}<br />{{ form.city(style="width: 80%;") }}<br />
{% for error in form.city.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
<p>
{{ form.country.label }}<br /> {{ form.country(style="width: 80%;") }}<br />
{% for error in form.country.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
{{ form.postcode.label }}<br />{{ form.postcode }}<br />
{% for error in form.postcode.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
{{ form.country.label }}<br /> {{ form.country }}<br />
{% for error in form.country.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
{{ form.phone.label }}<br /> {{ form.phone }}<br />
{% for error in form.phone.errors %}
@ -96,19 +96,19 @@ $("#{{ form.org_account.id }}").click(function() {
<div class="panel-heading">Данни за юридическо лице</div>
<div class="panel-body">
<p>
{{ form.org_companyname.label }}<br />{{ form.org_companyname }}<br />
{{ form.org_companyname.label }}<br />{{ form.org_companyname(style="width: 80%;") }}<br />
{% for error in form.org_companyname.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
{{ form.org_regaddress.label }}<br />{{ form.org_regaddress }}<br />
{{ form.org_regaddress.label }}<br />{{ form.org_regaddress(style="width: 80%;") }}<br />
{% for error in form.org_regaddress.errors %}
{{ error }}<br />
{% endfor %}
</p>
<p>
{{ form.org_responsible.label }}<br />{{ form.org_responsible }}<br />
{{ form.org_responsible.label }}<br />{{ form.org_responsible(style="width: 80%;") }}<br />
{% for error in form.org_responsible.errors %}
{{ error }}<br />
{% endfor %}

View file

@ -1,5 +1,16 @@
{% extends "base.html" %}
{% block scripts %}
{{ super() }}
<script type="text/javascript">
//$(function() {
// $('.confirm').click(function() {
// return window.confirm("Are you sure?");
// });
//});
</script>
{% endblock %}
{% block page_content %}
<script>
@ -14,8 +25,9 @@ addEventListener("DOMContentLoaded", function() {
button.addEventListener("click", function(e) {
// When a click happens, stop the button
// from submitting our form (if we have one)
e.preventDefault();
e.preventDefault();
if (window.confirm("Are you sure?")) {
var clickedButton = e.target;
var command = clickedButton.value;
var vmid = clickedButton.getAttribute('vmid');
@ -35,13 +47,13 @@ addEventListener("DOMContentLoaded", function() {
request.open("GET", "/vmanager/" + command + "/" + vmid, true);
// and then we send it off
request.send();
});
}
//alert("command " + command + " executed.");
}
});
}
}, true);
</script>
<style type="text/css">
.tg {border-collapse:collapse;border-spacing:0;}
.tg td{font-family:Arial, sans-serif;font-size:14px;padding:1px 1px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;}
@ -109,8 +121,8 @@ addEventListener("DOMContentLoaded", function() {
<br />
{% if status[deploy.machine_id] == 'running' %}
<button class="btn btn-default btn-info" onclick="window.open('/vmanager/vmvnc/{{ deploy.machine_id }}', '_blank');"><span class="glyphicon glyphicon-console" aria-hidden="true"></span> Console</button>
<button class="command command-vmshutdown btn btn-default btn-warning" value="vmshutdown" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-off" aria-hidden="true"></span> Shutdown</button>
<button class="command command-vmstop btn btn-default btn-danger" value="vmstop" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-alert" aria-hidden="true"></span> Force Stop</button>
<button class="confirm command command-vmshutdown btn btn-default btn-warning" value="vmshutdown" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-off" aria-hidden="true"></span> Shutdown</button>
<button class="confirm command command-vmstop btn btn-default btn-danger" value="vmstop" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-alert" aria-hidden="true"></span> Force Stop</button>
{% else %}
<button class="command command-vmstart btn btn-default btn-success" value="vmstart" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-play" aria-hidden="true"></span> Start</button>
{% endif %}

View file

@ -58,7 +58,7 @@ def deploy(product_id=None):
# return redirect(url_for('uinvoice.addfunds'))
if current_user.name is None:
flash('Моля обновете информацията за профила')
flash('Please update profile info for this operation.')
return redirect(url_for('uinvoice.profile'))
page = { 'title': 'Deploy' }
@ -123,17 +123,29 @@ def deploy(product_id=None):
@login_required
def dashboard():
deployments = current_user.inv_deployments.order_by(Deployment.date_created.desc()).all()
inv_deployments = []
inv_deploycubeids = []
inv_deploynames = []
for invcls in deployments:
if invcls.enabled == True:
inv_deployments.extend([invcls.machine_id])
#if not inv_deployments:
# return render_template('vmanager/empty_dashboard.html')
inv_deploycubeids.extend([invcls.machine_id])
inv_deploynames.extend([invcls.machine_alias])
contracts = current_user.inv_contracts.order_by(Contract.date_created.desc()).all()
inv_contracts = []
for invcls in contracts:
if invcls.enabled == True:
inv_contracts.extend([invcls.template])
domains = current_user.inv_domains.order_by(Domain.date_created.desc()).all()
inv_domains = []
for invcls in domains:
if invcls.enabled == True:
inv_domains.extend([invcls.fqdn])
#extract rrd and status from the deployments
rrd = {}
statuses = {}
for cubeid in inv_deployments:
for cubeid in inv_deploycubeids:
rrd[cubeid] = {}
try:
query = contact_proxmaster({}, 'vmrrd', cubeid)
@ -155,20 +167,7 @@ def dashboard():
send_email(current_app.config['MAIL_USERNAME'], 'Cube {} is unreachable'.format(cubeid),
'vmanager/email/adm_unreachable', user=current_user, cubeid=cubeid )
contracts = current_user.inv_contracts.order_by(Contract.date_created.desc()).all()
#inv_contracts = []
#for invcls in contracts:
# if invcls.enabled == True:
# inv_contracts.extend([invcls.template])
domains = current_user.inv_domains.order_by(Domain.date_created.desc()).all()
#inv_domains = []
#for invcls in domains:
# if invcls.enabled == True:
# inv_domains.extend([invcls.fqdn])
#print(statuses)
current_app.logger.info('[{}] deployments: {}, domains: {}, services: {}'.format(current_user.email, inv_deploynames, inv_contracts, inv_domains ))
return render_template('vmanager/dashboard.html', rrd=rrd, status=statuses, inv_deployments=deployments, inv_contracts=contracts, inv_domains=domains)
@vmanager.route('/vmvnc/<int:vmid>')

3
babel.cfg Normal file
View file

@ -0,0 +1,3 @@
[python: **.py]
[jinja2: **/templates/**.html]
extensions=jinja2.ext.autoescape,jinja2.ext.with_

36
before_upgrade.txt Normal file
View file

@ -0,0 +1,36 @@
alembic==0.9.0
Babel==2.3.4
blinker==1.4
click==6.7
dnspython==1.15.0
dnspython3==1.15.0
dominate==2.3.1
Flask==0.12
Flask-Babel==0.11.1
Flask-Bootstrap==3.3.7.1
Flask-Login==0.4.0
Flask-Mail==0.9.1
Flask-Migrate==2.0.3
Flask-Script==2.0.5
Flask-SQLAlchemy==2.2
Flask-WTF==0.14.2
gunicorn==19.7.0
iso3166==0.8
itsdangerous==0.24
Jinja2==2.9.5
Mako==1.0.6
MarkupSafe==0.23
onetimepass==1.0.1
pkg-resources==0.0.0
psycopg2==2.6.2
PyQRCode==1.2.1
python-editor==1.0.3
pytz==2016.10
requests==2.13.0
schedule==0.4.2
six==1.10.0
sortedcontainers==1.5.7
SQLAlchemy==1.1.6
visitor==0.1.3
Werkzeug==0.11.15
WTForms==2.1

View file

@ -1,17 +1,39 @@
Flask
Flask-Login
Flask-Bootstrap
Flask-WTF
Flask-Script
Flask-Migrate
Flask-Mail
Flask-SQLAlchemy
Flask-Babel
psycopg2
itsdangerous
requests
sortedcontainers
iso3166
pyqrcode
onetimepass
schedule
alembic==0.9.2
Babel==2.4.0
blinker==1.4
click==6.7
dnspython==1.15.0
dnspython3==1.15.0
dominate==2.3.1
Flask==0.12.2
Flask-Babel==0.11.2
Flask-Bootstrap==3.3.7.1
Flask-Login==0.4.0
Flask-Mail==0.9.1
Flask-Migrate==2.0.3
Flask-Script==2.0.5
Flask-SQLAlchemy==2.2
Flask-WTF==0.14.2
gunicorn==19.7.1
iso3166==0.8
itsdangerous==0.24
Jinja2==2.9.6
Mako==1.0.6
MarkupSafe==1.0
onetimepass==1.0.1
pkg-resources==0.0.0
psycopg2==2.7.1
PyQRCode==1.2.1
python-dateutil==2.6.0
python-editor==1.0.3
pytz==2017.2
requests==2.14.2
schedule==0.4.2
six==1.10.0
sortedcontainers==1.5.7
SQLAlchemy==1.1.10
visitor==0.1.3
Werkzeug==0.12.2
WTForms==2.1
facepy==1.0.8
appdirs==1.4.3