Make AWS Glue table descriptions editable

.gitignore

@@ -1,4 +1,5 @@
 .idea/*
 */.idea/*
 *.pyc
-settings.py
+settings.py
+venv/*

dashboard/plugins/table_descriptions/README.md

@@ -1,8 +1,10 @@
 # Table descriptions View
 
-The table descriptions tab displays table and column descriptions (comments). This can be useful for stakeholders to better understand your data structure and search for particular information. The default implementation takes them from AWS glue which stores comments added during table creation. Since not all tables have to have comments provided, this tab is fully optional.
-
-It's possible to set a custom data provider which reads the table descriptions from a different source than AWS glue. This is controlled by the `TABLE_DESCRIPTION_SERVICE` setting. 
+Table descriptions tab displays table and column descriptions (comments). 
+This can be useful for stakeholders to better understand your data structure and search for particular information. 
+The implementation takes them from AWS glue which stores comments added during table creation. 
+Since not all tables have to have comments provided, this tab is fully optional.  
+Comments for tables and fields are editable using the UI and will be commited to AWS Glue data store.  
 
 Table descriptions plugin requires valid [tables.yaml](../tables/tables.yaml.template) to be
 provided to provide a subset of the tables to be displayed.

dashboard/plugins/table_descriptions/__init__.py

@@ -1,12 +1,20 @@
 from flask import Blueprint
 from flask import render_template
 from flask import current_app as app
+from flask import request
 from dashboard.utils import load_data_provider
 
+import boto3
+import json
+import datetime
+
 base_path = '/table_descriptions'
 tab_name = 'Table descriptions'
 plugin = Blueprint('table_descriptions', __name__, template_folder='templates')
 
+def splitpart (value, index, char = ','):
+    return value.split(char)[index]
+
 DEFAULT_CACHE = 300
 def get_descriptions():
     cache_ttl = app.config.get('TABLE_DESCRIPTIONS_TTL', DEFAULT_CACHE)
@@ -22,11 +30,80 @@ def get_descriptions():
 
     descriptions = load_data_provider(classname, params).get_table_descriptions()
     app.cache.set('table_descriptions', descriptions, timeout=cache_ttl)
+    app.jinja_env.filters['splitpart'] = splitpart
     return descriptions
 
 @plugin.route('/')
 def index():
     if not 'TABLE_DESCRIPTIONS_SERVICE' in app.config or app.table_data_provider is None:
         return render_template('no_config.html', filename='TABLE_DESCRIPTIONS_SERVICE')
 
-    return render_template('table_descriptions/index.html', tables=get_descriptions())
+    return render_template('table_descriptions/index.html', tables=get_descriptions())
+
+"""
+Commits an update to glue with a new description of a given column from a submitted form
+"""
+@plugin.route('/post_update_column')
+def update_column():
+    # fetch data from the submitted form
+    column_description = request.args.get('column_description')
+    example = request.args.get('example')
+    table_name = request.args.get('table_name')
+    namespace = request.args.get('namespace')
+    col_to_update = request.args.get('column')
+    # we connect to aws glue and fetch old metadata of the column
+    glue = boto3.client('glue',
+                            aws_access_key_id=app.config['ATHENA_USAGE_PARAMS']['aws_access_key_id'],
+                            aws_secret_access_key=app.config['ATHENA_USAGE_PARAMS']['aws_secret_access_key'],
+                            region_name=app.config['ATHENA_USAGE_PARAMS']['region_name'])
+    table_data = glue.get_table(DatabaseName = namespace, Name = table_name)
+    # update the description in the metadata of the column and prepare data to send to glue
+    columns_data = table_data['Table']['StorageDescriptor']['Columns']
+    now = datetime.datetime.now()
+    comment_dict = dict([('column_description', column_description),
+                         ('example', example),
+                         ('last_update_timestamp', now.strftime('%Y-%m-%dT%H:%M:%S'))])
+    for col in columns_data:
+        if col['Name'] == col_to_update:
+            col['Comment'] = json.dumps(comment_dict, ensure_ascii=False)
+    update_dict = table_data['Table']
+    keys_to_delete = ['DatabaseName', 'UpdateTime', 'IsRegisteredWithLakeFormation']
+    for key in keys_to_delete:
+        del(update_dict[key])
+    # submit an update request to glue
+    glue.update_table(DatabaseName='test', TableInput=update_dict)
+    app.cache.clear()
+    return index()
+
+"""
+Commits an update to glue with a new description of a given table from a submitted form
+"""
+@plugin.route('/post_update_table')
+def update_table():
+
+    #table_description=opis+audience+sizingu&namespace=test&table_name=audience_sizing
+
+    # fetch data from the submitted form
+    table_description = request.args.get('table_description')
+    table_name = request.args.get('table_name')
+    namespace = request.args.get('namespace')
+    # we connect to aws glue and fetch old metadata of the column
+    glue = boto3.client('glue',
+                            aws_access_key_id=app.config['ATHENA_USAGE_PARAMS']['aws_access_key_id'],
+                            aws_secret_access_key=app.config['ATHENA_USAGE_PARAMS']['aws_secret_access_key'],
+                            region_name=app.config['ATHENA_USAGE_PARAMS']['region_name'])
+    table_data = glue.get_table(DatabaseName = namespace, Name = table_name)['Table']
+    # update the description in the metadata of the table and prepare data to send to glue
+    parameters = table_data['Parameters']
+    now = datetime.datetime.now()
+    comment_dict = dict([('table_description', table_description),
+                         ('last_update_timestamp', now.strftime('%Y-%m-%dT%H:%M:%S'))])
+    parameters['comment'] = json.dumps(comment_dict, ensure_ascii=False)
+    update_dict = table_data
+    keys_to_delete = ['DatabaseName', 'UpdateTime', 'IsRegisteredWithLakeFormation']
+    for key in keys_to_delete:
+        del (update_dict[key])
+    # submit an update request to glue
+    glue.update_table(DatabaseName='test', TableInput=update_dict)
+    app.cache.clear()
+    return index()

dashboard/plugins/table_descriptions/dataproviders.py

@@ -2,6 +2,7 @@
 from timeit import default_timer as timer
 
 import boto3
+import json
 
 from flask import current_app as app
 
@@ -53,6 +54,8 @@ def get_tables_for_db(self, db, tables):
                 columns = [{'name': column['Name'],
                             'description': column.get('Comment', ''),
                             'is_partition': False,
+                            #TODO: fill this
+                            'example': 'ToDoImport',
                             'type': column.get('Type')}
                            for column in table['StorageDescriptor']['Columns']]
 
@@ -83,9 +86,27 @@ def get_table_descriptions(self):
             for db, tables in self.db_tables.items():
                 futures.append((db, app.async_request_executor.submit(self.get_tables_for_db, db, tables)))
 
-            result = dict()
+            tables_metadata = dict()
             for db, tables in futures:
                 for table in tables.result():
-                    result['{}.{}'.format(db, table['name'])] = table
-
-        return result
+                    tables_metadata['{}.{}'.format(db, table['name'])] = table
+
+        for table_name, metadata in tables_metadata.items():
+            try:
+                td_dict = json.loads(metadata['description'])
+                metadata['description'] = td_dict['table_description']
+                metadata['last_description_update'] = td_dict['last_update_timestamp']
+            except ValueError as err:
+                metadata['previous_description_update'] = ''
+
+            for column in metadata['columns']:
+                try:
+                    cd_dict = json.loads(column['description'])
+                    column['description'] = cd_dict['column_description']
+                    column['last_description_update'] = cd_dict['last_update_timestamp']
+                    column['example'] = cd_dict['example']
+                except ValueError as err:
+                    column['last_description_update'] = ''
+                    column['example'] = ''
+
+        return tables_metadata

dashboard/plugins/table_descriptions/templates/table_descriptions/index.html

@@ -1,6 +1,7 @@
 {% extends 'layouts/base.html' %}
 {% import 'macros/common.html' as common with context %}
 
+
 {% block autorefresh %}{% endblock %}
 
 {% block title %}Table {% endblock %}
@@ -12,7 +13,8 @@
 
 {% block body %}
 
-    <link href="{{ url_for('static', filename='css/descriptions.css') }}" rel="stylesheet">
+    <link href="{{ url_for('static', filename='css/descriptions.css') }}"
+          rel="stylesheet">
 
     <div id="tooling">
         <div id="search">
@@ -22,43 +24,129 @@
             </div>
         </div>
         <div id="legend">
-            Legend: <span class="cui-layers" aria-hidden="true"></span> Partition column
+            Legend: <span class="cui-layers" aria-hidden="true"></span>
+            Partition column
         </div>
     </div>
 
     <div id="table-container">
         <div class="row mt-3 table-header">
             <div class="col-3">
-                <span id="all-toggle" class="toggle"><span class="cui-chevron-right" aria-hidden="true"></span></span>
+                <span id="all-toggle" class="toggle"><span
+                        class="cui-chevron-right"
+                        aria-hidden="true"></span></span>
                 Database | Table name
             </div>
             <div class="col-9">Description</div>
         </div>
         {% for table_name, details in tables.items() %}
+            {% set edit_columns_loop = loop.index %}
+
             <div id="{{ table_name }}" class="table-data">
-                <div id="header-{{ table_name }}" class="row column-header-section">
+                <div id="header-{{ table_name }}"
+                     class="row column-header-section">
                     <div class="col-3">
                         <span style="white-space: nowrap;">
-                            <span id="toggle-{{ table_name }}" class="toggle row-toggle">
-                                <span class="cui-chevron-right" aria-hidden="true"></span>
+                            <span id="toggle-{{ table_name }}"
+                                  class="toggle row-toggle">
+                                <span class="cui-chevron-right"
+                                      aria-hidden="true"></span>
                             </span>
                             {{ table_name }}
                         </span>
                     </div>
-                    <div class="col-9">{{ details.description }}</div>
+                    <div class="col-9"
+                         id="table-description-div-{{ edit_columns_loop }}">
+                        <div id="table-description-edit-div-{{ '%s' % edit_columns_loop }}"
+                             style="display: none;">
+                            <form action="/table_descriptions/post_update_table">
+                                <input type="text" name="table_description"
+                                       value="{{ details.description }}">
+                                <input type="hidden" name="namespace"
+                                       value="{{ table_name | splitpart(0, '.') }}">
+                                <input type="hidden" name="table_name"
+                                       value="{{ table_name | splitpart(1, '.') }}">
+                                <input type="submit" value="Submit">
+                            </form>
+                        </div>
+                        <div id="table-description-text-div-{{ edit_columns_loop }}" style="display: block">
+                            {{ details.description }}
+                            <input type="image"
+                                   id="button-edit-table-description-{{ '%s' % edit_columns_loop }}"
+                                   src="http://icons.iconarchive.com/icons/custom-icon-design/mono-general-2/256/edit-icon.png"
+                                   alt="Image"
+                                   style="width: 15px; height: 15px">
+                            <script>
+                                document.getElementById("button-edit-table-description-{{ '%s' % edit_columns_loop }}").onclick = function () {
+                                    document.getElementById("table-description-edit-div-{{ '%s' % edit_columns_loop }}").style.display = "block";
+                                    document.getElementById("table-description-text-div-{{ '%s' % edit_columns_loop }}").style.display = "none";
+                                };
+                            </script>
+                            </input>
+                        </div>
+                    </div>
                 </div>
-                <div class="row column-data-section" id="columns_{{ table_name }}">
+                <div class="row column-data-section"
+                     id="columns_{{ table_name }}">
                     <div class="row table-columns-header">
                         <div class="col-3">Column</div>
                         <div class="col-3">Type</div>
-                        <div class="col-6">Description</div>
+                        <div class="col-3">Description</div>
+                        <div class="col-3">Example</div>
                     </div>
                     {% for column in details.columns %}
                         <div class="row column-data">
-                            <div class="col-3">{{ column.name }} {% if column.is_partition %}<span class="cui-layers" aria-hidden="true"></span>{% endif %}</div>
-                            <div class="col-3">{{ column.type }}</div>
-                            <div class="col-6">{{ column.description }}</div>
+                            <div class="col-3">{{ column.name }}
+                                {% if column.is_partition %}
+                                    <span class="cui-layers"
+                                          aria-hidden="true"></span>{% endif %}
+                            </div>
+                            <div class="col-3">{{ column.type }} </div>
+                            <div class="col-3">{{ column.description }} <br><i>
+                                {% if column.last_description_update %}
+                                    {{ column.last_description_update }} {% else %}
+                                    Sourced from SQL {% endif %} </i></div>
+                            <div class="col-3">{{ column.example }} </div>
+                            <div class="col-3 button-edit"
+                                 id="button-edit-{{ '%s' % edit_columns_loop }}-{{ '%s' % loop.index }}">
+                                <input type="image"
+                                       id="button-edit-{{ '%s' % edit_columns_loop }}-{{ '%s' % loop.index }}"
+                                       src="http://icons.iconarchive.com/icons/custom-icon-design/mono-general-2/256/edit-icon.png"
+                                       alt="Image"
+                                       style="width: 15px; height: 15px">
+                                <script>
+                                    document.getElementById("button-edit-{{ '%s' % edit_columns_loop }}-{{ '%s' % loop.index }}").onclick = function () {
+                                        document.getElementById("form-update-column-{{ '%s' % edit_columns_loop }}-{{ '%s' % loop.index }}").style.display = "block";
+                                    };
+                                </script>
+                                </input>
+                            </div>
                         </div>
+                        <form action="/table_descriptions/post_update_column"
+                              id="form-update-column-{{ '%s' % edit_columns_loop }}-{{ '%s' % loop.index }}"
+                              style="display: none;">
+                            <div class="row column-data">
+                                <div class="col-3">
+                                    <input type="submit" value="Submit">
+                                </div>
+                                <div class="col-3"></div>
+                                <div class="col-3">
+                                    <input type="text"
+                                           name="column_description"
+                                           value="{{ column.description }}">
+                                </div>
+                                <div class="col-3">
+                                    <input type="text" name="example"
+                                           value="{{ column.example }}">
+                                </div>
+                                <input type="hidden" name="namespace"
+                                       value="{{ table_name | splitpart(0, '.') }}">
+                                <input type="hidden" name="table_name"
+                                       value="{{ table_name | splitpart(1, '.') }}">
+                                <input type="hidden" name="column"
+                                       value="{{ column.name }}">
+                            </div>
+                        </form>
                     {% endfor %}
                 </div>
             </div>