simple news crawler from fb

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2015-2016 deflax
+Copyright (c) 2015-2017 deflax.net
 
 This software is provided 'as-is', without any express or implied
 warranty. In no event will the authors be held liable for any damages

app/__init__.py

@@ -34,6 +34,9 @@ babel.init_app(app)
 from .main import main as main_blueprint
 app.register_blueprint(main_blueprint)
 
+from .news import news as news_blueprint
+app.register_blueprint(news_blueprint, url_prefix='/news')
+
 from .auth import auth as auth_blueprint
 app.register_blueprint(auth_blueprint, url_prefix='/auth')
 
@@ -60,7 +63,7 @@ class CustomJSONEncoder(JSONEncoder):
 
 app.json_encoder = CustomJSONEncoder
 
-#if app.debug:
+#TODO: if app.debug:
 if not app.debug:
     import logging
     from logging.handlers import RotatingFileHandler
@@ -69,7 +72,7 @@ if not app.debug:
     file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
     app.logger.addHandler(file_handler)
     app.logger.setLevel(logging.DEBUG)
-    app.logger.info('Proxadmin started.')
+    #app.logger.info('Proxadmin started.')
 
 @app.errorhandler(403)
 def forbidden(e):

app/news/__init__.py

@@ -0,0 +1,3 @@
+from flask import Blueprint
+news = Blueprint('news', __name__)
+from . import routes

app/news/routes.py

@@ -0,0 +1,40 @@
+from flask import render_template, abort, redirect, url_for, abort, flash, request, current_app, make_response, g
+from . import news
+
+from functools import lru_cache
+import requests
+import json
+from facepy import GraphAPI
+
+@lru_cache(maxsize=8)
+def get_fb_token():
+    payload = {'grant_type': 'client_credentials', 'client_id': current_app.config['FB_APP_ID'], 'client_secret': current_app.config['FB_APP_SECRET']}
+    response = requests.post('https://graph.facebook.com/oauth/access_token?', params = payload)
+    fbjson = json.loads(response.text)
+    fbtoken = fbjson['access_token']
+    current_app.logger.info("FB Token: {}".format(fbtoken))
+    return fbtoken
+
+@news.route("/all", methods=['GET'])
+def all():
+    graph = GraphAPI(get_fb_token())
+    #current_app.logger.info(get_fb_token.cache_info())
+    page_id = current_app.config['FB_PAGE_ID']
+    datas = graph.get(page_id+'/posts?fields=id,message,created_time,link,icon', page=True, retry=3)
+    posts = []
+    for data in datas:
+        posts.append(data)
+    content=posts[0]['data']
+    return render_template('news/facebook.html', result=content)
+
+@news.route('/latest')
+def latest():
+    graph = GraphAPI(get_fb_token())
+    page_id = current_app.config['FB_PAGE_ID']
+    latest = graph.get(page_id+'/posts?limit=1&fields=id,message,created_time,link', page=True, retry=3)
+    posts = []
+    for data in latest:
+        posts.append(data)
+    content=posts[0]['data']
+    return render_template('news/latest.html', result=content)
+

app/settings/routes.py

@@ -51,7 +51,7 @@ def profile():
     form.twofactor.data = current_user.twofactor
 
     wallet = "%.2f" % round(current_user.wallet, 3)
-    current_app.logger.info('wallet: ' + wallet)
+    current_app.logger.info('[{}] wallet: {}'.format(current_user.email, wallet))
 
     return render_template('settings/profile.html', page=page, form=form, wallet=wallet)
 

app/templates/_formhelpers.html

@@ -1,12 +0,0 @@
-{% macro render_field(field) %}
-  <dt>{{ field.label }}
-  <dd>{{ field(**kwargs)|safe }}
-  {% if field.errors %}
-    <ul class=errors>
-    {% for error in field.errors %}
-      <li>{{ error }}</li>
-    {% endfor %}
-    </ul>
-  {% endif %}
-  </dd>
-{% endmacro %}

app/templates/_slide.html

@@ -1,57 +0,0 @@
-{% extends "base.html" %}
-
-{% block page_content %}
-<h1>Index</h1>
-
-<div id="myCarousel" class="carousel slide" data-ride="carousel">
-  <!-- Indicators -->
-  <ol class="carousel-indicators">
-    <li data-target="#myCarousel" data-slide-to="0" class="active"></li>
-    <li data-target="#myCarousel" data-slide-to="1"></li>
-    <li data-target="#myCarousel" data-slide-to="2"></li>
-    <li data-target="#myCarousel" data-slide-to="3"></li>
-  </ol>
-
-  <!-- Wrapper for slides -->
-  <div class="carousel-inner" role="listbox">
-    <div class="item active">
-      <img src="static/images/1.jpg" alt="Chania">
-    </div>
-
-    <div class="item">
-      <img src="static/images/2.jpg" alt="Chania">
-    </div>
-
-    <div class="item">
-      <img src="static/images/3.jpg" alt="Flower">
-    </div>
-
-    <div class="item">
-      <img src="static/images/4.jpg" alt="Flower">
-    </div>
-  </div>
-
-  <!-- Left and right controls -->
-  <a class="left carousel-control" href="#myCarousel" role="button" data-slide="prev">
-    <span class="glyphicon glyphicon-chevron-left" aria-hidden="true"></span>
-    <span class="sr-only">Previous</span>
-  </a>
-  <a class="right carousel-control" href="#myCarousel" role="button" data-slide="next">
-    <span class="glyphicon glyphicon-chevron-right" aria-hidden="true"></span>
-    <span class="sr-only">Next</span>
-  </a>
-</div>
-
-  <div class="row">
-    <div class="col-md-4">
-      <p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
-    </div>
-    <div class="col-md-4">
-      <p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
-    </div>
-    <div class="col-md-4">
-      <p>Yr umami selfies Carles DIY, pop-up Tonx meggings stumptown freegan street art Vice ethnic. Pickled gastropub lo-fi polaroid, ennui selvage meh Tumblr organic iPhone kale chips narwhal Echo Park. Tonx literally distillery Pitchfork McSweeney's semiotics. Stumptown YOLO fanny pack bespoke, kitsch Carles gastropub vegan. Biodiesel ennui church-key McSweeney's, selvage hoodie Brooklyn 90's lomo. Quinoa photo booth cliche semiotics. Roof party Etsy ethnic, fashion axe mlkshk 8-bit paleo.</p>
-    </div>
-  </div>
-
-{% endblock %}

app/templates/base.html

@@ -6,8 +6,6 @@
 
 {% block title %}Cloud Builder - Datapoint.bg{% endblock %}
 
-{% block html_attribs %} lang="en"{% endblock %}
-
 {% block head %}
 {{ super() }}
     <link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}" type="image/x-icon">

app/templates/footer_index.html

@@ -7,19 +7,13 @@
 				<p>tel: +359 (0) 32 398 295 <br /> email: <a href="mailto:office@datapoint.bg">office@datapoint.bg</a></p>
 			</div><!--/one four-->
 			
-			<div class="one_four">
-				<h4>Latest Tweet</h4>
-				<p><a href="#">@datapoint</a> Продавам пирони. Изгодно.</p>
-				<span class="tweet_day">2 Days ago</span>
-			</div><!--/one four-->
-			
 			<div class="one_four">
 				<h4>Datapoint</h4>
 				<p>We are a dedicated team working to bring you the best hosting in the business. 
 				We are passionate about what we do, and why we do it.<br> <a href="#">Read More…</a></p>
 			</div><!--/one four-->
 			
-			<div class="one_four last_col">
+			<div class="one_four">
 				<h4>Newsletter Signup</h4>
 				<p>Join our newsletter to stay informed on our latest updates and sales.</p>
 				<form action="#" method="post">
@@ -29,6 +23,11 @@
 					</div><!--/form field-->
 				</form>
 			</div><!--/one four-->
+
+			<div class="one_four last_col" id="newsmenu">
+                           <!--#include virtual="https://www.datapoint.bg/news/latest" -->
+			</div><!--/one four-->
+	
 			
 		</div><!--/page wrap-->
 	</div><!--/footer cols-->

app/templates/main/index.html

@@ -42,8 +42,15 @@
     };
     makeBSS('.bss-slides', opts);
     </script>
+
+    <script type="text/javascript">    
+    $(document).ready(function() {
+        $('#newsmenu').load('https://www.datapoint.bg/news/latest');
+});
+    </script>
 {% endblock %}
 
+
 {% block page_content %}
     <!-- 
     <div class="row">

app/templates/news/facebook.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+
+{% block page_content %}
+<h1>News</h1>
+  <div class="row">
+    <ul id="navigation">
+    {% for item in result %}
+       <li><a target="_blank" href="{{ item['link'] }}">{{ item['message'] }}</a> {{ item['created_time'] }}<br />
+    </li><br />
+    {% endfor %}
+    </ul>
+  </div>
+
+{% endblock %}

app/templates/news/latest.html

@@ -0,0 +1,8 @@
+{% block page_content %}
+    <h4>News</h4>
+    {% for item in result %}
+    <p><a target="_blank" href="{{ item['link'] }}">{{ item['message'] }}</a></p>
+    <span class="tweet_day">{{ item['created_time'] }}</span>
+    {% endfor %}<br />
+    <a href="/news/all">>>></a>
+{% endblock %}

app/templates/settings/profile.html

@@ -41,42 +41,42 @@ $("#{{ form.org_account.id }}").click(function() {
       <div class="panel panel-default">
         <div class="panel-heading">Данни на профила</div>
         <div class="panel-body">
-<form method="POST" action="{{ url_for('settings.profile') }}">
+<form method="POST" action="{{ url_for('settings.profile') }}"> 
     <p>
-        {{ form.name.label }}<br />{{ form.name }}<br />
+        {{ form.name.label }}<br />{{ form.name(style="width: 80%;") }}<br />
         {% for error in form.name.errors %}
             {{ error }}<br />
         {% endfor %}
     </p>
 
     <p>
-        {{ form.address.label }}<br /> {{ form.address }}<br />
+        {{ form.address.label }}<br /> {{ form.address(style="width: 80%;") }}<br />
         {% for error in form.address.errors %}
             {{ error }}<br />
         {% endfor %}
     </p>
 
     <p>
-        {{ form.city.label }}<br />{{ form.city }}<br />
+        {{ form.city.label }}<br />{{ form.city(style="width: 80%;") }}<br />
         {% for error in form.city.errors %}
             {{ error }}<br />
         {% endfor %}
     </p>
 
-    <p>
+   <p>
+       {{ form.country.label }}<br /> {{ form.country(style="width: 80%;") }}<br />
+       {% for error in form.country.errors %}
+           {{ error }}<br />
+       {% endfor %}
+   </p>
+
+   <p>
         {{ form.postcode.label }}<br />{{ form.postcode }}<br />
         {% for error in form.postcode.errors %}
             {{ error }}<br />
         {% endfor %}
     </p>
 
-    <p>
-        {{ form.country.label }}<br /> {{ form.country }}<br />
-        {% for error in form.country.errors %}
-            {{ error }}<br />
-        {% endfor %}
-    </p>
-
     <p>
         {{ form.phone.label }}<br /> {{ form.phone }}<br />
         {% for error in form.phone.errors %}
@@ -96,19 +96,19 @@ $("#{{ form.org_account.id }}").click(function() {
         <div class="panel-heading">Данни за юридическо лице</div>
         <div class="panel-body">
     <p>
-        {{ form.org_companyname.label }}<br />{{ form.org_companyname }}<br />
+        {{ form.org_companyname.label }}<br />{{ form.org_companyname(style="width: 80%;") }}<br />
         {% for error in form.org_companyname.errors %}
             {{ error }}<br />
         {% endfor %}
     </p>
     <p>
-        {{ form.org_regaddress.label }}<br />{{ form.org_regaddress }}<br />
+        {{ form.org_regaddress.label }}<br />{{ form.org_regaddress(style="width: 80%;") }}<br />
         {% for error in form.org_regaddress.errors %}
             {{ error }}<br />
         {% endfor %}
     </p>
     <p>
-        {{ form.org_responsible.label }}<br />{{ form.org_responsible }}<br />
+        {{ form.org_responsible.label }}<br />{{ form.org_responsible(style="width: 80%;") }}<br />
         {% for error in form.org_responsible.errors %}
             {{ error }}<br />
         {% endfor %}

app/templates/vmanager/dashboard.html

@@ -1,5 +1,16 @@
 {% extends "base.html" %}
 
+{% block scripts %}
+{{ super() }}
+<script type="text/javascript">
+//$(function() {
+//    $('.confirm').click(function() {
+//        return window.confirm("Are you sure?");
+//    });
+//});
+</script>
+{% endblock %}
+
 {% block page_content %}
 
 <script>
@@ -14,8 +25,9 @@ addEventListener("DOMContentLoaded", function() {
     button.addEventListener("click", function(e) {
       // When a click happens, stop the button
       // from submitting our form (if we have one)
-      e.preventDefault();
+    e.preventDefault();
 
+    if (window.confirm("Are you sure?")) {
       var clickedButton = e.target;
       var command = clickedButton.value;
       var vmid = clickedButton.getAttribute('vmid');
@@ -35,13 +47,13 @@ addEventListener("DOMContentLoaded", function() {
       request.open("GET", "/vmanager/" + command + "/" + vmid, true);
       // and then we send it off
       request.send();
-    });
+      //alert("command " + command + " executed.");
-  }
+   }
+  });
+ }
 }, true);
-
 </script>
 
-
 <style type="text/css">
 .tg  {border-collapse:collapse;border-spacing:0;}
 .tg td{font-family:Arial, sans-serif;font-size:14px;padding:1px 1px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;}
@@ -109,8 +121,8 @@ addEventListener("DOMContentLoaded", function() {
             <br />
           {% if status[deploy.machine_id] == 'running' %}
             <button class="btn btn-default btn-info" onclick="window.open('/vmanager/vmvnc/{{ deploy.machine_id }}', '_blank');"><span class="glyphicon glyphicon-console" aria-hidden="true"></span> Console</button>
-            <button class="command command-vmshutdown btn btn-default btn-warning" value="vmshutdown" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-off" aria-hidden="true"></span> Shutdown</button>
+            <button class="confirm command command-vmshutdown btn btn-default btn-warning" value="vmshutdown" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-off" aria-hidden="true"></span> Shutdown</button>
-            <button class="command command-vmstop btn btn-default btn-danger" value="vmstop" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-alert" aria-hidden="true"></span> Force Stop</button>
+            <button class="confirm command command-vmstop btn btn-default btn-danger" value="vmstop" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-alert" aria-hidden="true"></span> Force Stop</button>
           {% else %}             
             <button class="command command-vmstart btn btn-default btn-success" value="vmstart" vmid="{{ deploy.machine_id }}"><span class="glyphicon glyphicon-play" aria-hidden="true"></span> Start</button>
           {% endif %}

app/vmanager/routes.py

@@ -58,7 +58,7 @@ def deploy(product_id=None):
     #    return redirect(url_for('uinvoice.addfunds'))
 
     if current_user.name is None:
-        flash('Моля обновете информацията за профила')
+        flash('Please update profile info for this operation.')
         return redirect(url_for('uinvoice.profile'))
         
     page = { 'title': 'Deploy' }
@@ -123,17 +123,29 @@ def deploy(product_id=None):
 @login_required
 def dashboard():
     deployments = current_user.inv_deployments.order_by(Deployment.date_created.desc()).all()
-    
+    inv_deploycubeids = []
-    inv_deployments = []
+    inv_deploynames = []
     for invcls in deployments:
         if invcls.enabled == True:
-            inv_deployments.extend([invcls.machine_id])
+            inv_deploycubeids.extend([invcls.machine_id])
-    #if not inv_deployments:
+            inv_deploynames.extend([invcls.machine_alias])
-    #    return render_template('vmanager/empty_dashboard.html')
 
+    contracts = current_user.inv_contracts.order_by(Contract.date_created.desc()).all()
+    inv_contracts = []
+    for invcls in contracts:
+        if invcls.enabled == True:
+            inv_contracts.extend([invcls.template])
+    
+    domains = current_user.inv_domains.order_by(Domain.date_created.desc()).all()
+    inv_domains = []
+    for invcls in domains:
+        if invcls.enabled == True:
+            inv_domains.extend([invcls.fqdn])
+
+    #extract rrd and status from the deployments
     rrd = {}
     statuses = {}
-    for cubeid in inv_deployments:
+    for cubeid in inv_deploycubeids:
         rrd[cubeid] = {}
         try:
             query = contact_proxmaster({}, 'vmrrd', cubeid)
@@ -155,20 +167,7 @@ def dashboard():
             send_email(current_app.config['MAIL_USERNAME'], 'Cube {} is unreachable'.format(cubeid), 
                                           'vmanager/email/adm_unreachable', user=current_user, cubeid=cubeid )
 
-    contracts = current_user.inv_contracts.order_by(Contract.date_created.desc()).all()
+    current_app.logger.info('[{}] deployments: {}, domains: {}, services: {}'.format(current_user.email, inv_deploynames, inv_contracts, inv_domains ))
-    #inv_contracts = []
-    #for invcls in contracts:
-    #    if invcls.enabled == True:
-    #        inv_contracts.extend([invcls.template])
-    
-    domains = current_user.inv_domains.order_by(Domain.date_created.desc()).all()
-    #inv_domains = []
-    #for invcls in domains:
-    #    if invcls.enabled == True:
-    #        inv_domains.extend([invcls.fqdn])
-
-    #print(statuses)
-
     return render_template('vmanager/dashboard.html', rrd=rrd, status=statuses, inv_deployments=deployments, inv_contracts=contracts, inv_domains=domains)
 
 @vmanager.route('/vmvnc/<int:vmid>')

babel.cfg

@@ -0,0 +1,3 @@
+[python: **.py]
+[jinja2: **/templates/**.html]
+extensions=jinja2.ext.autoescape,jinja2.ext.with_

before_upgrade.txt

@@ -0,0 +1,36 @@
+alembic==0.9.0
+Babel==2.3.4
+blinker==1.4
+click==6.7
+dnspython==1.15.0
+dnspython3==1.15.0
+dominate==2.3.1
+Flask==0.12
+Flask-Babel==0.11.1
+Flask-Bootstrap==3.3.7.1
+Flask-Login==0.4.0
+Flask-Mail==0.9.1
+Flask-Migrate==2.0.3
+Flask-Script==2.0.5
+Flask-SQLAlchemy==2.2
+Flask-WTF==0.14.2
+gunicorn==19.7.0
+iso3166==0.8
+itsdangerous==0.24
+Jinja2==2.9.5
+Mako==1.0.6
+MarkupSafe==0.23
+onetimepass==1.0.1
+pkg-resources==0.0.0
+psycopg2==2.6.2
+PyQRCode==1.2.1
+python-editor==1.0.3
+pytz==2016.10
+requests==2.13.0
+schedule==0.4.2
+six==1.10.0
+sortedcontainers==1.5.7
+SQLAlchemy==1.1.6
+visitor==0.1.3
+Werkzeug==0.11.15
+WTForms==2.1

requirements.txt

@@ -1,17 +1,39 @@
-Flask
+alembic==0.9.2
-Flask-Login
+Babel==2.4.0
-Flask-Bootstrap
+blinker==1.4
-Flask-WTF
+click==6.7
-Flask-Script
+dnspython==1.15.0
-Flask-Migrate
+dnspython3==1.15.0
-Flask-Mail
+dominate==2.3.1
-Flask-SQLAlchemy
+Flask==0.12.2
-Flask-Babel
+Flask-Babel==0.11.2
-psycopg2
+Flask-Bootstrap==3.3.7.1
-itsdangerous
+Flask-Login==0.4.0
-requests
+Flask-Mail==0.9.1
-sortedcontainers
+Flask-Migrate==2.0.3
-iso3166
+Flask-Script==2.0.5
-pyqrcode
+Flask-SQLAlchemy==2.2
-onetimepass
+Flask-WTF==0.14.2
-schedule
+gunicorn==19.7.1
+iso3166==0.8
+itsdangerous==0.24
+Jinja2==2.9.6
+Mako==1.0.6
+MarkupSafe==1.0
+onetimepass==1.0.1
+pkg-resources==0.0.0
+psycopg2==2.7.1
+PyQRCode==1.2.1
+python-dateutil==2.6.0
+python-editor==1.0.3
+pytz==2017.2
+requests==2.14.2
+schedule==0.4.2
+six==1.10.0
+sortedcontainers==1.5.7
+SQLAlchemy==1.1.10
+visitor==0.1.3
+Werkzeug==0.12.2
+WTForms==2.1
+facepy==1.0.8
+appdirs==1.4.3