diff --git a/README.md b/README.md index 0f8389b..ade98dd 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,10 @@ The tool is intended to be use directly online (still pending the final URL) so * Go to the src folder cd `src` * Run it using the following script: `tools/start`. Leave the window open, the application should be running. 6. Create an initial report - * Open a new Terminal window, leaving the other open, and run `curl -d '' "http://localhost:8080/_ah/cmd/create_report?year=2011&month=7&day=15"` + * Open a new Terminal window, leaving the other open, + * Initialize fusion tables: curl "http://localhost:8080/_ah/cmd/fusion_tables_names" + * Create an unclosed report curl -d '' "http://localhost:8080/_ah/cmd/create_report?year=2011&month=7&day=15" + * If you'd like, create an closed report curl -d '' "http://localhost:8080/_ah/cmd/create_report?year=2011&month=8&day=15&fyear=2011&fmonth=9&fday=15&assetid=SAD_VALIDATED/SAD_2010_05" 7. Start using the app. * You should now be able to go to http://localhost:8080 and start using the application locally. * When loggin in dont forget to set yourself as admin. diff --git a/src/application/api.py b/src/application/api.py index 856580f..904f6b8 100755 --- a/src/application/api.py +++ b/src/application/api.py @@ -14,6 +14,8 @@ from flask import jsonify, request, abort, Response from app import app import settings +from report_types import ReportType, CSVReportType, KMLReportType +from kml import path_to_kml from models import Area, Note, Report, StatsStore, FustionTablesNames from ee import NDFI, EELandsat, Stats @@ -44,9 +46,10 @@ #TODO: this function needs a huge refactor -@app.route('/api/v0/stats//') +@app.route('/api/v0/stats/
//') +@app.route('/api/v0/stats/
/') @app.route('/api/v0/stats/
') -def stats(table, zone=None): +def stats(table, zone=None, format="csv"): reports = request.args.get('reports', None) if not reports: @@ -56,67 +59,33 @@ def stats(table, zone=None): except ValueError: logging.error("bad format for report id") abort(400) + + this_report = ReportType.factory(format) + this_report.init(zone) + this_report.write_header() - f = StringIO() - csv_file = csv.writer(f) + logging.info("table id is %s ", table) + logging.info("and we see %s ", FustionTablesNames.all().filter('table_id =', table).fetch(1)) + logging.info("and zone %s ", zone) + logging.info("and format %s ", format) - table_names = FustionTablesNames.all().filter('table_id =', table).fetch(1)[0].as_dict() + reports = [Report.get_by_id(x) for x in reports] + for r in reports: + if not r: + logging.error("report not found") + abort(404) + + stats = this_report.get_stats(r, table) - # return the stats for each zone - if not zone: - csv_file.writerow(('report_id', 'start_date', 'end_date','zone_id', 'deforested', 'degraded')) - reports = [Report.get_by_id(x) for x in reports] - for r in reports: - if not r: - logging.error("report not found") - abort(404) - st = StatsStore.get_for_report(str(r.key())) - if not st: - logging.error("no stats for report") - abort(404) - stats = st.for_table(table) - for s in stats: - name = table_names.get(s['id'], s['id']) - csv_file.writerow((str(r.key().id()), - r.start.isoformat(), - r.end.isoformat(), - name, - s['def'], - s['deg'])) - - else: - csv_file.writerow(('report_id', 'start_date', 'end_date', 'deforested', 'degraded')) - reports = [Report.get_by_id(x) for x in reports] - for r in reports: - if not r: - abort(404) - report_id = str(r.key()) - st = StatsStore.get_for_report(report_id) - - if not st: - logging.error("no cached stats for %s" % report_id) - abort(404) - - stats = st.table_accum(table, zone) - if not stats: - logging.error("no stats for %s" % report_id) - abort(404) - - csv_file.writerow((str(r.key().id()), - r.start.isoformat(), - r.end.isoformat(), - stats['def'], - stats['deg'])) - - return Response(f.getvalue(), - headers={ - "Content-Disposition": "attachment; filename=\"report_%s.csv\"" % table - }, - mimetype='text/csv') - - -@app.route('/api/v0/stats/polygon/csv') -def polygon_stats_csv(): + for s in stats: + this_report.write_row(r, s, table) + + this_report.write_footer() + return this_report.response("report_%s" % table) + + +@app.route('/api/v0/stats/polygon/') +def polygon_stats(format=None): reports = request.args.get('reports', None) if not reports: abort(400) @@ -131,30 +100,29 @@ def polygon_stats_csv(): except ValueError: logging.error("can't find some report") abort(404) + #TODO: test if polygon is ccw # exchange lat, lon -> lon, lat polygon = json.loads(request.args.get('polygon', None)) + polygon.append(polygon[0]) + logging.info(polygon) + logging.info(path_to_kml([polygon])) if not polygon: abort(404) ee = Stats() normalized_poly = [(coord[1], coord[0]) for coord in polygon] stats = ee.get_stats_for_polygon([(str(r.key().id()), r.assetid) for r in reports], [normalized_poly]) + + this_report = ReportType.factory(format) + this_report.init("custom polygon") try: - f = StringIO() - csv_file = csv.writer(f) - csv_file.writerow(('report_id', 'start_date', 'end_date', 'deforested', 'degraded')) + this_report.write_header() for i,s in enumerate(stats): r = reports[i] - csv_file.writerow((str(r.key().id()), - r.start.isoformat(), - r.end.isoformat(), - s['def'], - s['deg'])) - return Response(f.getvalue(), - headers={ - "Content-Disposition": "attachment; filename=\"polygon.csv\"" - }, - mimetype='text/csv') + this_report.write_row(r, s, None, path_to_kml([polygon])) + + this_report.write_footer() + return this_report.response("report_polygon") except (KeyError, ValueError, IndexError): abort(404) diff --git a/src/application/ft.py b/src/application/ft.py index 54b3e6b..35f7426 100755 --- a/src/application/ft.py +++ b/src/application/ft.py @@ -51,7 +51,6 @@ def create_table(self, table): def sql(self, sql): logging.debug("FT:SQL: %s" % sql) r = self.client.query(sql) - logging.debug("-> %s" % r) return r diff --git a/src/application/generate_reports.py b/src/application/generate_reports.py index 6ee51dc..bd3058b 100644 --- a/src/application/generate_reports.py +++ b/src/application/generate_reports.py @@ -7,34 +7,36 @@ import urllib2 from time_utils import month_range -assets_id = ['SAD_VALIDATED/SAD_2009_08', -'SAD_VALIDATED/SAD_2009_09', -'SAD_VALIDATED/SAD_2009_10', -'SAD_VALIDATED/SAD_2009_11', -'SAD_VALIDATED/SAD_2009_12', -'SAD_VALIDATED/SAD_2010_01', -'SAD_VALIDATED/SAD_2010_02', -'SAD_VALIDATED/SAD_2010_05', -'SAD_VALIDATED/SAD_2010_06', -'SAD_VALIDATED/SAD_2010_07', -'SAD_VALIDATED/SAD_2010_08', -'SAD_VALIDATED/SAD_2010_09', -'SAD_VALIDATED/SAD_2010_10', -'SAD_VALIDATED/SAD_2010_11', -'SAD_VALIDATED/SAD_2010_12', +assets_id = [ +#'SAD_VALIDATED/SAD_2009_08', +#'SAD_VALIDATED/SAD_2009_09', +#'SAD_VALIDATED/SAD_2009_10', +#'SAD_VALIDATED/SAD_2009_11', +#'SAD_VALIDATED/SAD_2009_12', +#'SAD_VALIDATED/SAD_2010_01', +#'SAD_VALIDATED/SAD_2010_02', +#'SAD_VALIDATED/SAD_2010_05', +#'SAD_VALIDATED/SAD_2010_06', +#'SAD_VALIDATED/SAD_2010_07', +#'SAD_VALIDATED/SAD_2010_08', +#'SAD_VALIDATED/SAD_2010_09', +#'SAD_VALIDATED/SAD_2010_10', +#'SAD_VALIDATED/SAD_2010_11', +#'SAD_VALIDATED/SAD_2010_12', 'SAD_VALIDATED/SAD_2011_01', -'SAD_VALIDATED/SAD_2011_02', -'SAD_VALIDATED/SAD_2011_03', -'SAD_VALIDATED/SAD_2011_04', -'SAD_VALIDATED/SAD_2011_05', -'SAD_VALIDATED/SAD_2011_06', -'SAD_VALIDATED/SAD_2011_07', -'SAD_VALIDATED/SAD_2011_08', -'SAD_VALIDATED/SAD_2011_09'] - +#'SAD_VALIDATED/SAD_2011_02', +#'SAD_VALIDATED/SAD_2011_03', +#'SAD_VALIDATED/SAD_2011_04', +#'SAD_VALIDATED/SAD_2011_05', +#'SAD_VALIDATED/SAD_2011_06', +#'SAD_VALIDATED/SAD_2011_07', +#'SAD_VALIDATED/SAD_2011_08', +#'SAD_VALIDATED/SAD_2011_09' +] print "assetid,report_id" -for r in assets_id[-1:]: +#for r in assets_id[-1:]: +for r in assets_id: first, last = month_range(*reversed(map(int, r.split('_')[-2:]))) url = "http://%s/_ah/cmd/create_report?year=%d&month=%d&day=%d&assetid=%s&fyear=%d&fmonth=%d&fday=%d""" % (sys.argv[1], first.year, first.month, first.day, r, last.year, last.month, last.day) print url diff --git a/src/application/models.py b/src/application/models.py index 30b5b0a..053d902 100755 --- a/src/application/models.py +++ b/src/application/models.py @@ -512,10 +512,11 @@ def table_accum(self, table, zone=None): if not table_stats: logging.info("no stats for %s on %s" % (table, self.report_id)) return None - return { + return [{ + 'id': zone, 'def': reduce(operator.add, map(float, (x['def'] for x in table_stats))), 'deg': reduce(operator.add, map(float, (x['deg'] for x in table_stats))) - } + }] class FustionTablesNames(db.Model): table_id = db.StringProperty() @@ -523,4 +524,3 @@ class FustionTablesNames(db.Model): def as_dict(self): return json.loads(self.json) - diff --git a/src/application/report_types.py b/src/application/report_types.py new file mode 100755 index 0000000..8f04e7e --- /dev/null +++ b/src/application/report_types.py @@ -0,0 +1,209 @@ +""" +report_types.py + +Classes for describing different report formats. + +""" + +import logging +import csv +from application import settings +from google.appengine.ext import deferred +from datetime import datetime, date +from dateutil.relativedelta import relativedelta +from ft import FT +from flask import Response, abort, request +from StringIO import StringIO +from models import FustionTablesNames, StatsStore + + +class ReportType(object): + + zone = None + + def init(self, zone): + self.f.truncate(0) + self.zone = zone + + def set_zone(self): + self.zone = zone + + def write_row(self, report, table=None, kml=None): + raise NotImplementedError + + def write_header(self): + raise NotImplementedError + + def write_footer(self): + raise NotImplementedError + + def value(self): + raise NotImplementedError + + def response(self, file_name): + raise NotImplementedError + + def get_stats(self, report, table): + report_id = str(report.key()) + st = StatsStore.get_for_report(report_id) + if not st: + logging.error("no cached stats for %s" % report_id) + abort(404) + + if self.zone: + stats = st.table_accum(table, self.zone) + if not stats: + logging.error("no stats for %s" % report_id) + abort(404) + else: + stats = st.for_table(table) + if not stats: + logging.error("no stats for %s" % report_id) + abort(404) + return stats + + def get_polygon_name(table, id): + all_table_name = FustionTablesNames.all() + filtered_table_names = all_table_names.filter('table_id =', table) + table_names=filtered_table_names.fetch(1)[0].as_dict() + name = table_names.get(id, id) + + @staticmethod + def factory(format): + if (format == "kml"): + return KMLReportType() + else: + return CSVReportType() + + + +class CSVReportType(ReportType): + + f = StringIO() + csv_file = csv.writer(f) + + def write_header(self): + if self.zone: + self.csv_file.writerow(('report_id', 'start_date', 'end_date', + 'deforested', 'degraded')) + else: + self.csv_file.writerow(('report_id', 'start_date', 'end_date', + 'zone_id', 'deforested', 'degraded')) + + def write_footer(self): + pass + + def write_row(self, report, stats, table=None, kml=None): + name = None + + if table and not self.zone: + name = get_polygon_name(table, stats['id']) + + if name: + self.csv_file.writerow((str(report.key().id()), + report.start.isoformat(), + report.end.isoformat(), + name, + stats['def'], + stats['deg'])) + else: + self.csv_file.writerow((str(report.key().id()), + report.start.isoformat(), + report.end.isoformat(), + stats['def'], + stats['deg'])) + + + def value(self): + return self.f.getvalue() + + def response(self, file_name): + result = self.value() + self.f.truncate(0) + return Response(result, + headers={ + "Content-Disposition": "attachment; filename=\"" + file_name + + ".csv\"" + }, + mimetype='text/csv') + +class KMLReportType(ReportType): + + f = StringIO() + + def write_header(self): + self.f.write("") + self.f.write("") + self.f.write("") + self.f.write("") + + def write_footer(self): + self.f.write("") + self.f.write("") + + def write_row(self, report, stats, table=None, kml=None): + name = None + + if table: + name = get_polygon_name(table, stats['id']) + kml = self.kml(table, stats['id']) + else: + name = "Custom Polygon" + + description = self.description(name, stats) + + self.f.write("") + self.f.write("#transGreenPoly") + self.f.write("" + name + "") + self.f.write("" + description + "") + self.f.write(kml) + self.f.write("") + + def value(self): + return self.f.getvalue() + + def response(self, file_name): + result = self.value() + self.f.truncate(0) + return Response(result, + headers={ + "Content-Disposition": "attachment; filename=\"" + file_name + + ".kml\"" + }, + mimetype='text/kml') + + def kml(self, table, row_id): + cl = FT(settings.FT_CONSUMER_KEY, + settings.FT_CONSUMER_SECRET, + settings.FT_TOKEN, + settings.FT_SECRET) + + #TODO: do this better + if (table == 1568452): + id = 'ex_area' + else: + id = 'name' + + info = cl.sql("select geometry from %s where %s = %s" % + (table, id, row_id)) + polygon = info.split('\n')[1] + polygon = polygon.replace("\"", "") + return polygon + + def description(self, name, stats): + desc = "" + desc += "" + desc += "

" + name + desc += "

Deforestation: " + desc += str(stats['def']) + "km2
Degradation: " + str(stats['deg']) + desc += "km2
]]>" + return desc + + + + + diff --git a/src/application/resources/report.py b/src/application/resources/report.py index ffec513..e6ee2fe 100755 --- a/src/application/resources/report.py +++ b/src/application/resources/report.py @@ -19,6 +19,7 @@ from google.appengine.api import memcache + class NDFIMapApi(Resource): """ resource to get ndfi map access data """ @@ -224,11 +225,6 @@ def rgb_mapid(self, report_id, id, r, g, b): return Response(json.dumps(mapid['data']), mimetype='application/json') - - - - - class PolygonAPI(Resource): def list(self, report_id, cell_pos): diff --git a/src/application/settings.py b/src/application/settings.py index 221c685..70858ea 100755 --- a/src/application/settings.py +++ b/src/application/settings.py @@ -24,8 +24,8 @@ if DEBUG: - FT_TABLE = 'areas_dev' - FT_TABLE_ID = '1556991' + FT_TABLE = 'imazon_testing' + FT_TABLE_ID = '2676501' else: app_id = app_identity.get_application_id() if app_id == 'imazon-sad-tool': diff --git a/src/content/templates/404.html b/src/content/templates/404.html index 6b8191c..d0bd85a 100755 --- a/src/content/templates/404.html +++ b/src/content/templates/404.html @@ -2,6 +2,6 @@ {% block text_box %}

404 - Not found.

-

The page have asked for does not exist. Please check the URL and try again, or go to home page ans start again

+

The page have asked for does not exist. Please check the URL and try again, or go to home page and start again

Go to gome page {% endblock %} diff --git a/src/index.yaml b/src/index.yaml index bc9a180..054eaaa 100644 --- a/src/index.yaml +++ b/src/index.yaml @@ -16,6 +16,11 @@ indexes: - name: added_on direction: desc +- kind: Report + properties: + - name: finished + - name: start + - kind: Report properties: - name: finished diff --git a/src/static/js/vis/report_dialog.js b/src/static/js/vis/report_dialog.js index 2148482..f56799c 100644 --- a/src/static/js/vis/report_dialog.js +++ b/src/static/js/vis/report_dialog.js @@ -37,7 +37,7 @@ var ReportDialog = Backbone.View.extend({ //input this.reports = this.options.reports; - this.formats = ['csv'];//, 'kml']; + this.formats = ['csv', 'kml']; $(document).bind('keydown', this.keyPress); @@ -119,18 +119,17 @@ var ReportDialog = Backbone.View.extend({ var url = '/api/v0/stats/'; if(this.custom) { if(this.custom.polygon) { - url += 'polygon/csv'; + url += 'polygon/' + this.selected_format; } else { - url += this.custom.table + '/' + this.custom.zone; + url += this.custom.table + '/' + this.selected_format + '/' + this.custom.zone; } } else { - url += this.region_selected.table; + url += this.region_selected.table + '/' + this.selected_format; } url += '?reports=' + reports.join(','); if(this.custom && this.custom.polygon) { url += '&polygon=' + encodeURI(JSON.stringify(this.custom.polygon)); } - //alert(url); console.log(url); window.open(url); }, diff --git a/src/static/js/vis/stats.js b/src/static/js/vis/stats.js index e312816..71e1ff8 100644 --- a/src/static/js/vis/stats.js +++ b/src/static/js/vis/stats.js @@ -9,7 +9,7 @@ var ReportStat = Backbone.Model.extend({ /* ======================================== - store all statictics in the cliend side. Take this class + store all statistics in the client side. Take this class as a memcache in client side ======================================== */ @@ -80,6 +80,11 @@ var PolygonStat = Backbone.Model.extend({ $.post(this.url() + '/csv', this.attributes ); + }, + get_kml: function() { + $.post(this.url() + '/kml', + this.attributes + ); } });