"""i18n_subsites plugin creates i18n-ized subsites of the default site This plugin is designed for Pelican 3.4 and later """ import os import six import logging import posixpath from copy import copy from itertools import chain from operator import attrgetter from collections import OrderedDict from contextlib import contextmanager from six.moves.urllib.parse import urlparse import gettext import locale from pelican import signals from pelican.generators import ArticlesGenerator, PagesGenerator from pelican.settings import configure_settings from pelican.contents import Draft # Global vars _MAIN_SETTINGS = None # settings dict of the main Pelican instance _MAIN_LANG = None # lang of the main Pelican instance _MAIN_SITEURL = None # siteurl of the main Pelican instance _MAIN_STATIC_FILES = None # list of Static instances the main Pelican instance _SUBSITE_QUEUE = {} # map: lang -> settings overrides _SITE_DB = OrderedDict() # OrderedDict: lang -> siteurl _SITES_RELPATH_DB = {} # map: (lang, base_lang) -> relpath # map: generator -> list of removed contents that need interlinking _GENERATOR_DB = {} _NATIVE_CONTENT_URL_DB = {} # map: source_path -> content in its native lang _LOGGER = logging.getLogger(__name__) @contextmanager def temporary_locale(temp_locale=None): '''Enable code to run in a context with a temporary locale Resets the locale back when exiting context. Can set a temporary locale if provided ''' orig_locale = locale.setlocale(locale.LC_ALL) if temp_locale is not None: locale.setlocale(locale.LC_ALL, temp_locale) yield locale.setlocale(locale.LC_ALL, orig_locale) def initialize_dbs(settings): '''Initialize internal DBs using the Pelican settings dict This clears the DBs for e.g. autoreload mode to work ''' global _MAIN_SETTINGS, _MAIN_SITEURL, _MAIN_LANG, _SUBSITE_QUEUE _MAIN_SETTINGS = settings _MAIN_LANG = settings['DEFAULT_LANG'] _MAIN_SITEURL = settings['SITEURL'] _SUBSITE_QUEUE = settings.get('I18N_SUBSITES', {}).copy() prepare_site_db_and_overrides() # clear databases in case of autoreload mode _SITES_RELPATH_DB.clear() _NATIVE_CONTENT_URL_DB.clear() _GENERATOR_DB.clear() def prepare_site_db_and_overrides(): '''Prepare overrides and create _SITE_DB _SITE_DB.keys() need to be ready for filter_translations ''' _SITE_DB.clear() _SITE_DB[_MAIN_LANG] = _MAIN_SITEURL # make sure it works for both root-relative and absolute main_siteurl = '/' if _MAIN_SITEURL == '' else _MAIN_SITEURL for lang, overrides in _SUBSITE_QUEUE.items(): if 'SITEURL' not in overrides: overrides['SITEURL'] = posixpath.join(main_siteurl, lang) _SITE_DB[lang] = overrides['SITEURL'] # default subsite hierarchy if 'OUTPUT_PATH' not in overrides: overrides['OUTPUT_PATH'] = os.path.join( _MAIN_SETTINGS['OUTPUT_PATH'], lang) if 'CACHE_PATH' not in overrides: overrides['CACHE_PATH'] = os.path.join( _MAIN_SETTINGS['CACHE_PATH'], lang) if 'STATIC_PATHS' not in overrides: overrides['STATIC_PATHS'] = [] if ('THEME' not in overrides and 'THEME_STATIC_DIR' not in overrides and 'THEME_STATIC_PATHS' not in overrides): relpath = relpath_to_site(lang, _MAIN_LANG) overrides['THEME_STATIC_DIR'] = posixpath.join( relpath, _MAIN_SETTINGS['THEME_STATIC_DIR']) overrides['THEME_STATIC_PATHS'] = [] # to change what is perceived as translations overrides['DEFAULT_LANG'] = lang def subscribe_filter_to_signals(settings): '''Subscribe content filter to requested signals''' for sig in settings.get('I18N_FILTER_SIGNALS', []): sig.connect(filter_contents_translations) def initialize_plugin(pelican_obj): '''Initialize plugin variables and Pelican settings''' if _MAIN_SETTINGS is None: initialize_dbs(pelican_obj.settings) subscribe_filter_to_signals(pelican_obj.settings) def get_site_path(url): '''Get the path component of an url, excludes siteurl also normalizes '' to '/' for relpath to work, otherwise it could be interpreted as a relative filesystem path ''' path = urlparse(url).path if path == '': path = '/' return path def relpath_to_site(lang, target_lang): '''Get relative path from siteurl of lang to siteurl of base_lang the output is cached in _SITES_RELPATH_DB ''' path = _SITES_RELPATH_DB.get((lang, target_lang), None) if path is None: siteurl = _SITE_DB.get(lang, _MAIN_SITEURL) target_siteurl = _SITE_DB.get(target_lang, _MAIN_SITEURL) path = posixpath.relpath(get_site_path(target_siteurl), get_site_path(siteurl)) _SITES_RELPATH_DB[(lang, target_lang)] = path return path def save_generator(generator): '''Save the generator for later use initialize the removed content list ''' _GENERATOR_DB[generator] = [] def article2draft(article): '''Transform an Article to Draft''' draft = Draft(article._content, article.metadata, article.settings, article.source_path, article._context) draft.status = 'draft' return draft def page2hidden_page(page): '''Transform a Page to a hidden Page''' page.status = 'hidden' return page class GeneratorInspector(object): '''Inspector of generator instances''' generators_info = { ArticlesGenerator: { 'translations_lists': ['translations', 'drafts_translations'], 'contents_lists': [('articles', 'drafts')], 'hiding_func': article2draft, 'policy': 'I18N_UNTRANSLATED_ARTICLES', }, PagesGenerator: { 'translations_lists': ['translations', 'hidden_translations'], 'contents_lists': [('pages', 'hidden_pages')], 'hiding_func': page2hidden_page, 'policy': 'I18N_UNTRANSLATED_PAGES', }, } def __init__(self, generator): '''Identify the best known class of the generator instance The class ''' self.generator = generator self.generators_info.update(generator.settings.get( 'I18N_GENERATORS_INFO', {})) for cls in generator.__class__.__mro__: if cls in self.generators_info: self.info = self.generators_info[cls] break else: self.info = {} def translations_lists(self): '''Iterator over lists of content translations''' return (getattr(self.generator, name) for name in self.info.get('translations_lists', [])) def contents_list_pairs(self): '''Iterator over pairs of normal and hidden contents''' return (tuple(getattr(self.generator, name) for name in names) for names in self.info.get('contents_lists', [])) def hiding_function(self): '''Function for transforming content to a hidden version''' hiding_func = self.info.get('hiding_func', lambda x: x) return hiding_func def untranslated_policy(self, default): '''Get the policy for untranslated content''' return self.generator.settings.get(self.info.get('policy', None), default) def all_contents(self): '''Iterator over all contents''' translations_iterator = chain(*self.translations_lists()) return chain(translations_iterator, *(pair[i] for pair in self.contents_list_pairs() for i in (0, 1))) def filter_contents_translations(generator): '''Filter the content and translations lists of a generator Filters out 1) translations which will be generated in a different site 2) content that is not in the language of the currently generated site but in that of a different site, content in a language which has no site is generated always. The filtering method bay be modified by the respective untranslated policy ''' inspector = GeneratorInspector(generator) current_lang = generator.settings['DEFAULT_LANG'] langs_with_sites = _SITE_DB.keys() removed_contents = _GENERATOR_DB[generator] for translations in inspector.translations_lists(): for translation in translations[:]: # copy to be able to remove if translation.lang in langs_with_sites: translations.remove(translation) removed_contents.append(translation) hiding_func = inspector.hiding_function() untrans_policy = inspector.untranslated_policy(default='hide') for (contents, other_contents) in inspector.contents_list_pairs(): for content in other_contents: # save any hidden native content first if content.lang == current_lang: # in native lang # save the native URL attr formatted in the current locale _NATIVE_CONTENT_URL_DB[content.source_path] = content.url for content in contents[:]: # copy for removing in loop if content.lang == current_lang: # in native lang # save the native URL attr formatted in the current locale _NATIVE_CONTENT_URL_DB[content.source_path] = content.url elif content.lang in langs_with_sites and untrans_policy != 'keep': contents.remove(content) if untrans_policy == 'hide': other_contents.append(hiding_func(content)) elif untrans_policy == 'remove': removed_contents.append(content) def install_templates_translations(generator): '''Install gettext translations in the jinja2.Environment Only if the 'jinja2.ext.i18n' jinja2 extension is enabled the translations for the current DEFAULT_LANG are installed. ''' if 'JINJA_ENVIRONMENT' in generator.settings: # pelican 3.7+ jinja_extensions = generator.settings['JINJA_ENVIRONMENT'].get( 'extensions', []) else: jinja_extensions = generator.settings['JINJA_EXTENSIONS'] if 'jinja2.ext.i18n' in jinja_extensions: domain = generator.settings.get('I18N_GETTEXT_DOMAIN', 'messages') localedir = generator.settings.get('I18N_GETTEXT_LOCALEDIR') if localedir is None: localedir = os.path.join(generator.theme, 'translations') current_lang = generator.settings['DEFAULT_LANG'] if current_lang == generator.settings.get('I18N_TEMPLATES_LANG', _MAIN_LANG): translations = gettext.NullTranslations() else: langs = [current_lang] try: translations = gettext.translation(domain, localedir, langs) except (IOError, OSError): _LOGGER.error(( "Cannot find translations for language '{}' in '{}' with " "domain '{}'. Installing NullTranslations.").format( langs[0], localedir, domain)) translations = gettext.NullTranslations() newstyle = generator.settings.get('I18N_GETTEXT_NEWSTYLE', True) generator.env.install_gettext_translations(translations, newstyle) def add_variables_to_context(generator): '''Adds useful iterable variables to template context''' context = generator.context # minimize attr lookup context['relpath_to_site'] = relpath_to_site context['main_siteurl'] = _MAIN_SITEURL context['main_lang'] = _MAIN_LANG context['lang_siteurls'] = _SITE_DB current_lang = generator.settings['DEFAULT_LANG'] extra_siteurls = _SITE_DB.copy() extra_siteurls.pop(current_lang) context['extra_siteurls'] = extra_siteurls def interlink_translations(content): '''Link content to translations in their main language so the URL (including localized month names) of the different subsites will be honored ''' lang = content.lang # sort translations by lang content.translations.sort(key=attrgetter('lang')) for translation in content.translations: relpath = relpath_to_site(lang, translation.lang) url = _NATIVE_CONTENT_URL_DB[translation.source_path] translation.override_url = posixpath.join(relpath, url) def interlink_translated_content(generator): '''Make translations link to the native locations for generators that may contain translated content ''' inspector = GeneratorInspector(generator) for content in inspector.all_contents(): interlink_translations(content) def interlink_removed_content(generator): '''For all contents removed from generation queue update interlinks link to the native location ''' current_lang = generator.settings['DEFAULT_LANG'] for content in _GENERATOR_DB[generator]: url = _NATIVE_CONTENT_URL_DB[content.source_path] relpath = relpath_to_site(current_lang, content.lang) content.override_url = posixpath.join(relpath, url) def interlink_static_files(generator): '''Add links to static files in the main site if necessary''' if generator.settings['STATIC_PATHS'] != []: return # customized STATIC_PATHS filenames = generator.context['filenames'] # minimize attr lookup relpath = relpath_to_site(generator.settings['DEFAULT_LANG'], _MAIN_LANG) for staticfile in _MAIN_STATIC_FILES: if staticfile.get_relative_source_path() not in filenames: staticfile = copy(staticfile) # prevent override in main site staticfile.override_url = posixpath.join(relpath, staticfile.url) generator.add_source_path(staticfile) def save_main_static_files(static_generator): '''Save the static files generated for the main site''' global _MAIN_STATIC_FILES # test just for current lang as settings change in autoreload mode if static_generator.settings['DEFAULT_LANG'] == _MAIN_LANG: _MAIN_STATIC_FILES = static_generator.staticfiles def update_generators(): '''Update the context of all generators Ads useful variables and translations into the template context and interlink translations ''' for generator in _GENERATOR_DB.keys(): install_templates_translations(generator) add_variables_to_context(generator) interlink_static_files(generator) interlink_removed_content(generator) interlink_translated_content(generator) def get_pelican_cls(settings): '''Get the Pelican class requested in settings''' cls = settings['PELICAN_CLASS'] if isinstance(cls, six.string_types): module, cls_name = cls.rsplit('.', 1) module = __import__(module) cls = getattr(module, cls_name) return cls def create_next_subsite(pelican_obj): '''Create the next subsite using the lang-specific config If there are no more subsites in the generation queue, update all the generators (interlink translations and removed content, add variables and translations to template context). Otherwise get the language and overrides for next the subsite in the queue and apply overrides. Then generate the subsite using a PELICAN_CLASS instance and its run method. Finally, restore the previous locale. ''' global _MAIN_SETTINGS if len(_SUBSITE_QUEUE) == 0: _LOGGER.debug( 'i18n: Updating cross-site links and context of all generators.') update_generators() _MAIN_SETTINGS = None # to initialize next time else: with temporary_locale(): settings = _MAIN_SETTINGS.copy() lang, overrides = _SUBSITE_QUEUE.popitem() settings.update(overrides) settings = configure_settings(settings) # to set LOCALE, etc. cls = get_pelican_cls(settings) new_pelican_obj = cls(settings) _LOGGER.debug(("Generating i18n subsite for language '{}' " "using class {}").format(lang, cls)) new_pelican_obj.run() # map: signal name -> function name _SIGNAL_HANDLERS_DB = { 'get_generators': initialize_plugin, 'article_generator_pretaxonomy': filter_contents_translations, 'page_generator_finalized': filter_contents_translations, 'get_writer': create_next_subsite, 'static_generator_finalized': save_main_static_files, 'generator_init': save_generator, } def register(): '''Register the plugin only if required signals are available''' for sig_name in _SIGNAL_HANDLERS_DB.keys(): if not hasattr(signals, sig_name): _LOGGER.error(( 'The i18n_subsites plugin requires the {} ' 'signal available for sure in Pelican 3.4.0 and later, ' 'plugin will not be used.').format(sig_name)) return for sig_name, handler in _SIGNAL_HANDLERS_DB.items(): sig = getattr(signals, sig_name) sig.connect(handler)