#! python # copyleft (c) Pierre-Jean Turpeau 04/2022 # # first inspired by https://news.ycombinator.com/item?id=30957181 import os import sys import shutil import re import yaml import json from datetime import datetime from markdown_it import MarkdownIt from mdit_py_plugins.anchors import anchors_plugin from mdit_py_plugins.front_matter import front_matter_plugin from mdit_py_plugins.wordcount import wordcount_plugin CONFIG = { # the title prefix to be used on all pages "page_title_prefix" : "pj", # dict of source_dir: dest_dir (=> generate into www_dir/dest_dir/) "content_dirs" : { "pierrejean": { "target_dir": "_output", "copy_extra_files" : True }, "notes": { "target_dir": "_output/notes", "copy_extra_files" : True }, "retro": { "target_dir": "_output/retro", "copy_extra_files" : True }, "demoscene": { "target_dir": "_output/demoscene", "copy_extra_files" : True }, "projects": { "target_dir": "_output/projects", "copy_extra_files" : True }, "projet-feau": { "target_dir": "_output/projet-feau", "copy_extra_files" : True }, "projet-feau/pierre-cantais": { "target_dir": "_output/projet-feau/pierre-cantais", "copy_extra_files" : True }, "x86-code": { "target_dir": "_output/x86-code", "copy_extra_files" : True }, "private": { "target_dir": "_output/private", "copy_extra_files" : True }, "lynx": { "target_dir": "_output/lynx", "copy_extra_files": True }, "sec": { "target_dir": "_output/sec", "copy_extra_files": True }, "health": { "target_dir": "_output/health", "copy_extra_files": True } # "www/revelation": "revelation", # "www/ASL": "ASL", # "www/babyloon": "babyloon", }, "template_dir": "pierrejean/_templates", # directory location to produce the generated html files "www_dir" : "_output", "blog_json": "_output/notes/notes.json", # regex to replace internal .md links by links to .html generated files "links_regex" : r"\[([éèôîàùûa-zA-Z0-9_\-\.,\"'\ \\\/]+?)\]\((?!http)(.*?)\.md\)", # the rule to substitute the regex groups for the links replacement "links_sub" : r"[\1](\2)", # "links_sub" : r"[\1](\2.html)", # the regular expression used to identify private files and directories. # private files and directories are excluded from the production. "private_regex" : r"^_.*$", "home_link" : "
/home
", # extensions to check for files to copy "as is" in the target directory "file_types_to_copy" : [ ".bat", ".cfg", ".css", ".docx", ".exe", ".gif", ".gz", ".htaccess", ".htpasswd", ".html", ".ima", ".jar", ".jpg", ".js", ".pdf", ".php", ".png", ".pptx", ".py", ".sh", ".txt", ".xlsx", ".zip", ".svg", ".sh3d", ".json" ], # can be a file or a directory "other_paths_to_copy" : { "scripts/gen.py" : "pub/s4g/", "scripts/sync.sh" : "pub" } } class s4pg: def __init__(self, config:dict): self._cfg = config self._private_regex = re.compile(config['private_regex']) self._links_regex = re.compile(config['links_regex']) self._links_sub = config['links_sub'] self._php_regex = re.compile(r"\<\!\-\- php: (.*) \-\-\>") self._plugin_regex = re.compile(r"\<\!\-\- addon: (.*)\(\) \-\-\>") self.sys_path = sys.path def _is_public(self, path:str)->bool: return self._private_regex.search(path) == None def generate(self): target_dir = os.path.normpath(self._cfg['www_dir']) if os.path.exists(target_dir): shutil.rmtree(target_dir) self.root_out_dir = target_dir self.template_dir = os.path.join(".", self._cfg['template_dir']) self.blog_entries = [] for src, conf in self._cfg['content_dirs'].items(): self._generate_from(src, conf['target_dir'], copy_extra_files=conf['copy_extra_files']) self._copy_other_files_or_dirs(target_dir) if len(self.blog_entries) > 0: blog_entries = sorted(self.blog_entries, reverse=True, key=lambda k: k['date']) with open(self._cfg['blog_json'], 'w', encoding='utf-8') as f: json.dump(blog_entries, f, ensure_ascii=False) print(f" ** {len(blog_entries)} blog notes pushed into {self._cfg['blog_json']}") def _generate_from(self, src_dir:str, target_dir:str, copy_extra_files:bool=True): md = ( MarkdownIt('commonmark' ,{'breaks':False,'html':True}) .use(anchors_plugin) .use(front_matter_plugin) .use(wordcount_plugin) .enable('table') ) for folder, subs, files in os.walk(src_dir): folder_basename = os.path.basename(folder) if self._is_public(folder_basename): print("\n* Looking into folder: "+folder) # copy the public directory structure out_dir = folder.replace(src_dir, target_dir) # os.makedirs(out_dir, exist_ok=True) else: continue # skip private directory folder_hierarchy = os.path.normpath(out_dir.replace(self.root_out_dir, '')) # relative path to the root dir (for home links) root_rel_path = os.path.relpath(self.root_out_dir, out_dir) for src_filename in files: src_filename = src_filename file_basename, file_extension = os.path.splitext(src_filename) src_file_path = os.path.join(folder, src_filename) dst_dir_path = folder.replace(src_dir, target_dir) module = None if self._is_public(src_filename): if file_extension == '.md': print(" Markdown found: "+src_filename, end="") if src_filename == 'citations.md': json_path = os.path.join(dst_dir_path, file_basename+'.json') self._build_citations_json(md, src_file_path, json_path) with open(src_file_path, 'r', encoding='utf-8') as src: text = src.read() text = self._links_regex.sub(self._links_sub, text) env = {} tokens = md.parse(text, env = env) metadata = {} if tokens[0].type == 'front_matter': metadata = dict(yaml.safe_load(tokens[0].content)) addon = metadata.get('addon') if addon: sys.path.append(src_dir) module = __import__(addon) func_list = self._plugin_regex.findall(text) print(f" [{addon}:", end="") for func_name in func_list: print(f" {func_name}", end="") func = getattr(module, func_name) regex_str = r"\<\!\-\- addon: "+re.escape(func_name)+r"\(\) \-\-\>" text = re.sub(regex_str, func(), text) print("]", end="") sys.path.pop() html = md.render(text) # skip unpublished .md documents if metadata.get('published', False) != True: print(); # to close to line... continue extension = metadata.get('extension', 'html') # output to .php when a php addon is used if self._php_regex.search(html) is not None: extension = 'php' html = self._php_regex.sub(r"", html) # retrieve the 'default' template - can be overloaded by the 'template' metadata template_file = os.path.join(self.template_dir, metadata.get('template', 'default')+'.html') with open(template_file, 'r') as tmpl: template = tmpl.read() html_filename = file_basename + "." + extension print(" => published as " + html_filename) homelink = self._cfg['home_link'].format(home_dir = root_rel_path) # TODO: get rid of short-title in existing pages... browser_title = metadata.get('short-title', None) if browser_title is None: browser_title = metadata.get('page_title', None) if browser_title is not None: title = self._cfg['page_title_prefix'] +"/ "+browser_title else: title = "/ ".join([self._cfg['page_title_prefix'], file_basename.replace('-', ' ')]) if metadata.get('blog', False) == True: blog_title = metadata.get('blog-title', '*none*') d = str(metadata.get('blog-date', '*none*')) blog_long_date = datetime(year=int(d[0:4]), month=int(d[5:7]), day=int(d[8:10])).strftime("%b %d, %Y") blog_date = d blog_tags = metadata.get('blog-tags', '') tags = ['#'+sub for sub in blog_tags.split()] blog_tags = ' '.join(tags) self.blog_entries.append({'date':blog_date, 'title':blog_title, 'tags':blog_tags, 'link':file_basename, 'longdate':blog_long_date}) # publish the output using basic python templating os.makedirs(out_dir, exist_ok=True) with open(os.path.join(out_dir, html_filename), 'w', encoding='utf-8') as f: f.write( template % { 'title': title, 'content': html, 'homelink': homelink, 'rootpath': root_rel_path, 'minutes': env["wordcount"]["minutes"] } ) # copy files "as is" iff extension match file_types_to_copy configuration elif copy_extra_files \ and (file_extension in self._cfg['file_types_to_copy'] \ or src_filename in self._cfg['file_types_to_copy']): dst_file_path = src_file_path.replace(src_dir, target_dir, 1) print(f" + copying {src_file_path} to {dst_file_path}") os.makedirs(os.path.dirname(dst_file_path), exist_ok=True) shutil.copyfile(src_file_path, dst_file_path) def _copy_other_files_or_dirs(self, target_dir:str): print(f"\n* Copying extra files/dirs") for src, dst in self._cfg['other_paths_to_copy'].items(): if os.path.exists(src) and os.path.isfile(src): dst_dir = os.path.join(target_dir, dst) print(f" + copying {src} to {dst_dir}") os.makedirs(dst_dir, exist_ok=True) shutil.copy(src, dst_dir) def _build_citations_json(self, md:object, citations_path:str, json_path:str): """ 1. parse citations.md file 2. each text line starting with '>' or ' >' is added to md_fragment 3. when line starts with '---', then it converts fragment to html 4. add the generated html to a new json entry 5. save citations.json once all the citations have been parsed. """ count = 0 md_fragment = '' citations = [] with open(citations_path, 'r', encoding='utf-8') as f: lines = f.readlines() for line in lines: if line.startswith("---") and md_fragment != '': html = md.render(md_fragment) citations.append(html) md_fragment = '' elif line.startswith('>') or line.startswith(' >'): md_fragment += line if len(citations) > 0: with open(json_path, 'w', encoding='utf-8') as f: json.dump(citations, f, ensure_ascii=False) print(f" ** {len(citations)} citations pushed into {json_path}") def main(): generator = s4pg(CONFIG) generator.generate() if __name__ == "__main__": main()