diff --git a/appengine_main.py b/appengine_main.py index c3fd5928132..c9b3f5494cd 100644 --- a/appengine_main.py +++ b/appengine_main.py @@ -18,7 +18,6 @@ import os import webapp2 import logging -import traceback import devsitePage import devsiteIndex import devsiteHelper @@ -28,20 +27,24 @@ DEFAULT_LANG = 'en' DEVENV = os.environ['SERVER_SOFTWARE'].startswith('Dev') USE_MEMCACHE = not DEVENV +SOURCE_PATH = os.path.join(os.path.dirname(__file__), 'src/content/') class FlushMemCache(webapp2.RequestHandler): def get(self): memcache.flush_all() self.response.out.write('Flushed') + class HomePage(webapp2.RequestHandler): def get(self): self.redirect('/web/', permanent=True) + class DevSiteRedirect(webapp2.RequestHandler): def get(self, path): self.redirect('https://developers.google.com/' + path, permanent=True) + class Framebox(webapp2.RequestHandler): def get(self, path): response = None @@ -58,8 +61,15 @@ def get(self, path): logging.info('200 ' + memcacheKey) self.response.out.write(response) + class DevSitePages(webapp2.RequestHandler): def get(self, path): + + if path.endswith('.html') or path.endswith('.md'): + redirectTo = '/web/' + os.path.splitext(path)[0] + self.redirect(redirectTo, permanent=True) + return + response = None langQS = self.request.get('hl', None) langCookie = self.request.cookies.get('hl') @@ -82,8 +92,11 @@ def get(self, path): if response is None: try: - if path.endswith('/') or path == '': + if os.path.isdir(os.path.join(SOURCE_PATH, 'en', path)): response = devsiteIndex.getPage(path, lang) + if (response is None) and (path.startswith('showcase') or + path.startswith('shows') or path.startswith('updates')): + response = devsiteIndex.getDirIndex(path) else: response = devsitePage.getPage(path, lang) @@ -108,7 +121,7 @@ def get(self, path): response = render('gae/500.tpl', context) logging.exception('500 ' + fullPath) self.response.set_status(500) - + self.response.out.write(response) diff --git a/devsiteHelper.py b/devsiteHelper.py index ab5245d009e..1515e3db95e 100644 --- a/devsiteHelper.py +++ b/devsiteHelper.py @@ -6,15 +6,10 @@ import logging import textwrap import urllib2 - from datetime import date, datetime from google.appengine.api import memcache -from google.appengine.ext.webapp.template import render SOURCE_PATH = os.path.join(os.path.dirname(__file__), 'src/content/') -DEVENV = os.environ['SERVER_SOFTWARE'].startswith('Dev') -USE_MEMCACHE = not DEVENV - def slugify(str): # Very simply slugify @@ -34,8 +29,7 @@ def getFromMemCache(memcacheKey): def setMemCache(memcacheKey, value, length=3600): try: - if USE_MEMCACHE: - memcache.set(memcacheKey, value, length) + memcache.set(memcacheKey, value, length) except Exception as e: logging.exception('Unable to cache to MemCache') @@ -149,10 +143,6 @@ def parseBookYaml(pathToBook, lang='en'): Returns: A dictionary with the parsed book. """ - memcacheKey = 'bookYAML-' + pathToBook - result = getFromMemCache(memcacheKey) - if result: - return result try: result = {} upperTabs = [] @@ -160,7 +150,6 @@ def parseBookYaml(pathToBook, lang='en'): bookYaml = yaml.load(readFile(pathToBook, lang)) for upperTab in bookYaml['upper_tabs']: upperTabs.append(expandBook(upperTab)) - setMemCache(memcacheKey, result, 60) return result except Exception as e: logging.exception('Error in parseBookYaml') @@ -219,13 +208,8 @@ def getLowerTabs(bookYaml): def getLeftNav(requestPath, bookYaml, lang='en'): - # Returns the left nav. If it's already been generated and stored in - # memcache, return that, otherwise, read the file then recursively - # build the tree using buildLeftNav. - memcacheKey = 'leftNav-' + requestPath - result = getFromMemCache(memcacheKey) - if result: - return result + # Returns the left nav. Read the file then recursively, then build the + # tree using buildLeftNav. whoops = '
An error occured while trying to parse and build the' whoops += ' left hand navigation. Check the error logs.' @@ -241,10 +225,9 @@ def getLeftNav(requestPath, bookYaml, lang='en'): result = '
\n' - setMemCache(memcacheKey, result) return result except Exception as e: - msg = ' - Unable to read or parse primary book.yaml: ' + pathToBook + msg = ' - Unable to read or parse primary book.yaml' logging.exception(msg) whoops += 'Exception occured.
' return whoops @@ -254,18 +237,6 @@ def buildLeftNav(bookYaml, lang='en'): # Recursively reads the book.yaml file and generates the navigation tree result = '' for item in bookYaml: - if 'include' in item: - ## TODO(petele): Remove this - ## leaving this in for a few weeks while I ensure it doesn't break - ## anything. - logging.error('***** INCLUDE - this should NOT happen.') - # try: - # include = readFile(item['include'], lang) - # include = yaml.load(include) - # result += buildLeftNav(include['toc']) - # except Exception as e: - # msg = ' - Unable to parsing embedded toc file: ' + item['include'] - # logging.exception(msg) if 'path' in item: result += 'tags with prettyprint enabled tags - content = re.sub(r'^(?m)', r'', content) - - # Get the page title from the markup. - titleRO = re.search(r'(.*?)<\/h1>', content) - if titleRO: - title = titleRO.group(1) - - gitHubEditUrl = 'https://github.com/google/WebFundamentals/blob/' - gitHubEditUrl += 'master/src/content/' - gitHubEditUrl += fileLocation.replace(SOURCE_PATH, '') - - gitHubIssueUrl = 'https://github.com/google/WebFundamentals/issues/' - gitHubIssueUrl += 'new?title=Feedback for: ' + title + ' [' - gitHubIssueUrl += lang + ']&body=' - gitHubIssueUrl += gitHubEditUrl - - # Renders the content into the template - response = render(template, { - 'title': title, - 'announcementBanner': announcementBanner, - 'lowerTabs': lowerTabs, - 'gitHubIssueUrl': gitHubIssueUrl, - 'gitHubEditUrl': gitHubEditUrl, - 'requestPath': requestPath.replace('/index', ''), - 'leftNav': leftNav, - 'content': content, - 'toc': toc, - 'dateUpdated': dateUpdated, - 'lang': lang, - 'footerPromo': devsiteHelper.getFooterPromo(), - 'footerLinks': devsiteHelper.getFooterLinkBox() - } - ) - break - - return response + return devsiteParseMD.parse(requestPath, fileLocation, content, lang) + if fileLocation.endswith('.html'): + return devsiteParseHTML.parse(requestPath, fileLocation, content, lang) + return None diff --git a/devsiteParseHTML.py b/devsiteParseHTML.py new file mode 100644 index 00000000000..663552075e4 --- /dev/null +++ b/devsiteParseHTML.py @@ -0,0 +1,126 @@ +import os +import re +import logging +import devsiteHelper +from google.appengine.ext.webapp.template import render + +SOURCE_PATH = os.path.join(os.path.dirname(__file__), 'src/content/') +UNSUPPORTED_TAGS = [ + r'{% link_sample_button .+%}', + r'{% include_code (.+)%}' +] + +def parse(requestPath, fileLocation, content, lang='en'): + template = 'gae/article.tpl' + + if content.find('') == -1: + return content + + # Isolate the + headStart = content.find('') + headEnd = content.find('') + head = content[headStart:headEnd].strip() + + # Isolate the + bodyStart = content.find('') + bodyEnd = content.rfind('') + body = content[bodyStart:bodyEnd].strip() + + dateUpdated = re.search(r"{# wf_updated_on:[ ]?(.*)[ ]?#}", content) + if dateUpdated is None: + dateUpdated = 'Unknown' + else: + dateUpdated = dateUpdated.group(1) + + # Remove any comments {# something #} + body = re.sub(r'{#.+?#}', '', body) + body = re.sub(r'{% comment %}.*?{% endcomment %}(?ms)', '', body) + + # Show warning for unsupported elements + for tag in UNSUPPORTED_TAGS: + if re.search(tag, body) is not None: + logging.error(' - Unsupported tag: ' + tag) + replaceWith = '' + body = re.sub(tag, replaceWith, body) + + # Show warning for template tags + if re.search('{{', body) is not None: + logging.warn(' - Warning: possible unescaped template tag') + + # Render any DevSite specific tags + body = devsiteHelper.renderDevSiteContent(body, lang) + + # Read the page title + title = re.search('
(.*?) ', head) + if title is None: + title = '** UNKNOWN TITLE **' + else: + title = title.group(1) + + # Read the book.yaml file and generate the left hand nav + bookPath = re.search('name=\"book_path\" value=\"(.*?)\"', head) + if bookPath is None: + logging.error('Unable to read book_path') + leftNav = 'Book not found.' + lowerTabs = '' + else: + bookPath = bookPath.group(1) + bookYaml = devsiteHelper.parseBookYaml(bookPath, lang) + leftNav = devsiteHelper.getLeftNav(requestPath, bookYaml) + lowerTabs = devsiteHelper.getLowerTabs(bookYaml) + + # Read the project.yaml file + projectPath = re.search('name=\"project_path\" value=\"(.*?)\"', head) + if bookPath is None: + logging.error('Unable to read project_path') + else: + projectPath = projectPath.group(1) + announcementBanner = devsiteHelper.getAnnouncementBanner(projectPath, lang) + + # Replacestags with prettyprint enabled tags + body = re.sub(r'^(?m)', r'', body) + + # Checks if the page should be displayed in full width mode + fullWidth = re.search('name=\"full_width\" value=\"true\"', head) + if fullWidth is not None: + template = 'gae/home.tpl' + + # # Build the table of contents & transform so it fits within DevSite + toc = '- TOC NYI - ' + # toc = md.toc + # toc = toc.strip() + # # Strips the outer wrapper and the page title from the doc + # toc = re.sub(r'(.*?(?s)', '', toc) + # # Add appropriate classes + # toc = re.sub(r'){2}(?s)', '', toc) + # toc = re.sub(r'
\s*\s*\s*', '
', toc) + # toc = re.sub(r'', '
- ', toc) + + gitHubEditUrl = 'https://github.com/google/WebFundamentals/blob/' + gitHubEditUrl += 'master/src/content/' + gitHubEditUrl += fileLocation.replace(SOURCE_PATH, '') + + gitHubIssueUrl = 'https://github.com/google/WebFundamentals/issues/' + gitHubIssueUrl += 'new?title=Feedback for: ' + title + ' [' + gitHubIssueUrl += lang + ']&body=' + gitHubIssueUrl += gitHubEditUrl + + # Renders the content into the template + return render(template, { + 'title': title, + 'head': head, + 'announcementBanner': announcementBanner, + 'lowerTabs': lowerTabs, + 'gitHubIssueUrl': gitHubIssueUrl, + 'gitHubEditUrl': gitHubEditUrl, + 'requestPath': requestPath.replace('/index', ''), + 'leftNav': leftNav, + 'content': body, + 'toc': toc, + 'dateUpdated': dateUpdated, + 'lang': lang, + 'footerPromo': devsiteHelper.getFooterPromo(), + 'footerLinks': devsiteHelper.getFooterLinkBox() + }) diff --git a/devsiteParseMD.py b/devsiteParseMD.py new file mode 100644 index 00000000000..49d15c138a4 --- /dev/null +++ b/devsiteParseMD.py @@ -0,0 +1,132 @@ +import os +import re +import yaml +import logging +import markdown +import devsiteHelper +from google.appengine.ext.webapp.template import render + +SOURCE_PATH = os.path.join(os.path.dirname(__file__), 'src/content/') +UNSUPPORTED_TAGS = [ + r'{% link_sample_button .+%}', + r'{% include_code (.+)%}' +] + +def parse(requestPath, fileLocation, content, lang='en'): + template = 'gae/article.tpl' + + dateUpdated = re.search(r"{# wf_updated_on:[ ]?(.*)[ ]?#}", content) + if dateUpdated is None: + logging.warn('Missing wf_updated_on tag.') + dateUpdated = 'Unknown' + else: + dateUpdated = dateUpdated.group(1) + + ## Injects markdown includes into the markdown as appropriate + includes = re.findall(r'^<<.+?\.md>>(?m)', content) + for includeTag in includes: + fileName = includeTag.replace('<<', '').replace('>>', '') + fileName = os.path.join(os.path.dirname(fileLocation), fileName) + include = devsiteHelper.readFile(fileName, lang) + if include is None: + include = 'Warning: Unable to find included markdown file.\n\n' + content = content.replace(includeTag, include) + + # Remove any comments {# something #} + content = re.sub(r'{#.+?#}', '', content) + content = re.sub(r'{% comment %}.*?{% endcomment %}(?ms)', '', content) + + # Show warning for unsupported elements + for tag in UNSUPPORTED_TAGS: + if re.search(tag, content) is not None: + logging.error(' - Unsupported tag: ' + tag) + replaceWith = '' + content = re.sub(tag, replaceWith, content) + + # Show warning for template tags + if re.search('{{', content) is not None: + logging.warn(' - Warning: possible unescaped template tag') + + # Render any DevSite specific tags + content = devsiteHelper.renderDevSiteContent(content, lang) + + # If it's a markdown file, parse it to HTML + content = re.sub(r'^Note: (.*?)\n^\n(?ms)', r'', content) + content = re.sub(r'^Caution: (.*?)\n^\n(?ms)', r'', content) + content = re.sub(r'^Warning: (.*?)\n^\n(?ms)', r'', content) + content = re.sub(r'^Key Point: (.*?)\n^\n(?ms)', r'', content) + content = re.sub(r'^Key Term: (.*?)\n^\n(?ms)', r'', content) + content = re.sub(r'^Objective: (.*?)\n^\n(?ms)', r'', content) + content = re.sub(r'^Success: (.*?)\n^\n(?ms)', r'', content) + content = re.sub(r'^Dogfood: (.*?)\n^\n(?ms)', r'', content) + + # Adds a set of markdown extensions available to us on DevSite + ext = [ + 'markdown.extensions.attr_list', # Adds support for {: #someid } + 'markdown.extensions.meta', # Removes the meta data from the top of the doc + 'markdown.extensions.toc', # Generate the TOC for the right side + 'markdown.extensions.tables', # Support for Markdown Tables + 'markdown.extensions.def_list', # Support for definition lists + 'markdown.extensions.extra' # Support for markdown='1' in tags + ] + md = markdown.Markdown(extensions=ext) + content = md.convert(content) + + # Reads the book.yaml file and generate the lefthand nav + if 'book_path' in md.Meta and len(md.Meta['book_path']) == 1: + bookPath = md.Meta['book_path'][0] + bookYaml = devsiteHelper.parseBookYaml(bookPath, lang) + + if 'project_path' in md.Meta and len(md.Meta['project_path']) == 1: + projectPath = md.Meta['project_path'][0] + + if 'full_width' in md.Meta and len(md.Meta['full_width']) == 1: + template = 'gae/home.tpl' + + # Build the table of contents & transform so it fits within DevSite + toc = md.toc + toc = toc.strip() + # Strips the outer wrapper and the page title from the doc + toc = re.sub(r'
(.*?(?s)', '', toc) + # Add appropriate classes + toc = re.sub(r'){2}(?s)', '', toc) + toc = re.sub(r'
\s*\s*\s*', '
', toc) + toc = re.sub(r'', '
- ', toc) + + # Replaces
tags with prettyprint enabled tags + content = re.sub(r'^(?m)', r'', content) + + # Get the page title from the markup. + titleRO = re.search(r'(.*?)<\/h1>', content) + if titleRO: + title = titleRO.group(1) + else: + title = ':(' + + gitHubEditUrl = 'https://github.com/google/WebFundamentals/blob/' + gitHubEditUrl += 'master/src/content/' + gitHubEditUrl += fileLocation.replace(SOURCE_PATH, '') + + gitHubIssueUrl = 'https://github.com/google/WebFundamentals/issues/' + gitHubIssueUrl += 'new?title=Feedback for: ' + title + ' [' + gitHubIssueUrl += lang + ']&body=' + gitHubIssueUrl += gitHubEditUrl + + # Renders the content into the template + return render(template, { + 'title': title, + 'announcementBanner': devsiteHelper.getAnnouncementBanner(projectPath, lang), + 'lowerTabs': devsiteHelper.getLowerTabs(bookYaml), + 'gitHubIssueUrl': gitHubIssueUrl, + 'gitHubEditUrl': gitHubEditUrl, + 'requestPath': requestPath.replace('/index', ''), + 'leftNav': devsiteHelper.getLeftNav(requestPath, bookYaml), + 'content': content, + 'toc': toc, + 'dateUpdated': dateUpdated, + 'lang': lang, + 'footerPromo': devsiteHelper.getFooterPromo(), + 'footerLinks': devsiteHelper.getFooterLinkBox() + }) diff --git a/devsiteParseYAML.py b/devsiteParseYAML.py new file mode 100644 index 00000000000..f0708208c13 --- /dev/null +++ b/devsiteParseYAML.py @@ -0,0 +1,168 @@ +import os +import yaml +import logging +import devsitePage +import devsiteHelper +from google.appengine.ext.webapp.template import render + +SOURCE_PATH = os.path.join(os.path.dirname(__file__), 'src/content') + + +def parse(requestPath, fileLocation, rawYaml, lang='en'): + body = '' + parsedYaml = yaml.load(rawYaml) + bookPath = parsedYaml['book_path'] + bookYaml = devsiteHelper.parseBookYaml(bookPath, lang) + projectPath = parsedYaml['project_path'] + page = parsedYaml['landing_page'] + rows = page['rows'] + title = 'Web' + banner = devsiteHelper.getAnnouncementBanner(projectPath, lang) + header = 'Generic Page Header Here' + customCss = '' + lowerTabs = devsiteHelper.getLowerTabs(bookYaml) + if 'custom_css_path' in page: + customCss = '' + if 'header' in page: + header = '
' + header += '' + if 'name' in page['header']: + title = page['header']['name'] + for row in rows: + sectionClass = ['devsite-landing-row'] + section = '' + header += '' + if 'description' in page['header']: + header += '' + header += '' + if 'buttons' in page['header']: + header += ' ' + header += '' + header += page['header']['description'] + header += '' + if 'classname' in row: + sectionClass.append(row['classname']) + numItems = None + if 'columns' in row: + numItems = len(row['columns']) + elif 'items' in row: + numItems = len(row['items']) + if numItems: + sectionClass.append('devsite-landing-row-' + str(numItems) + '-up') + if 'heading' in row: + section += ' ' + section = section.replace('[[SECTION_CLASSES]]', ' '.join(sectionClass)) + body += section + body = devsiteHelper.renderDevSiteContent(body, lang) + return render('gae/home.tpl', { + 'title': title, + 'announcementBanner': banner, + 'requestPath': requestPath, + 'lowerTabs': lowerTabs, + 'customcss': customCss, + 'header': header, + 'content': body, + 'lang': lang, + 'footerPromo': devsiteHelper.getFooterPromo(), + 'footerLinks': devsiteHelper.getFooterLinkBox() + } + ) + + +def parseIndexYamlItems(yamlItems): + result = '' + for yamlItem in yamlItems: + item = '' + section += row['heading'] + '
' + if 'items' in row: + section += '' + section += parseIndexYamlItems(row['items']) + section += '' + if 'columns' in row: + for column in row['columns']: + section += '' + if 'items' in column: + section += parseIndexYamlItems(column['items']) + section += '' + section += '' + itemClasses = ['devsite-landing-row-item'] + descriptionClasses = ['devsite-landing-row-item-description'] + link = None + + if 'path' in yamlItem: + link = '' + + if 'icon' in yamlItem: + if link: + item += link + if 'icon_name' in yamlItem['icon']: + item += '' + + if 'image_path' in yamlItem: + imgClass = 'devsite-landing-row-item-image' + if 'image_left' in yamlItem: + imgClass += ' devsite-landing-row-item-image-left' + item += '' + elif not 'youtube_id' in yamlItem: + itemClasses.append('devsite-landing-row-item-no-image') + + if 'description' in yamlItem: + item += ' ' + descriptionClasses.append('devsite-landing-row-item-icon-description') + if 'path' in yamlItem['icon']: + item += '' + descriptionClasses.append('devsite-landing-row-item-icon-description') + if link: + item += '' + item = item.replace('[[ITEM_CLASSES]]', ' '.join(itemClasses)) + item = item.replace('[[DESCRIPTION_CLASSES]]', ' '.join(descriptionClasses)) + + result += item + + return result + diff --git a/gae/article.tpl b/gae/article.tpl index 30206695ec8..912248d45dc 100644 --- a/gae/article.tpl +++ b/gae/article.tpl @@ -41,6 +41,9 @@ }' + if 'heading' in yamlItem: + if link: + item += link + item += '' + + if 'custom_html' in yamlItem: + item += devsiteHelper.renderDevSiteContent(yamlItem['custom_html']) + + if 'youtube_id' in yamlItem: + item += '' + item += yamlItem['heading'] + '
' + # item += '' + yamlItem['heading'] + '
' + if link: + item += '' + item += yamlItem['description'] + if 'buttons' in yamlItem: + item += ' ' + item += '' + item += '' + item += '' + + item += '{{ title }} + {% autoescape off %} + {{ head }} + {% endautoescape %}diff --git a/gae/home.tpl b/gae/home.tpl index 705076be39d..1bdebcb814e 100644 --- a/gae/home.tpl +++ b/gae/home.tpl @@ -32,6 +32,9 @@ margin-top: 48px; } + {% autoescape off %} + {{ head }} + {% endautoescape %}